From 5320379bf5429a724d5c3d70a96a767fface3acd Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 13 Nov 2024 17:29:17 +0530 Subject: [PATCH 001/105] Handle export data status panics (#1874) 1. Handling nil pointer and the index out-of-range issues while running export data status before, during, and after export data. 2. Handled panics in `get data-migration report`, when it was run before export was started. 3. Fixed a bug in the `export data status` for partitions case from PG where we club the statuses of all leaf table to root tables's entry, which was not showing the progress of partitions and was just showing the NOT-STARTED for the entire duration of export data and was showing the DONE one it is completed. 4. Fixed the output of the export data status for the Debezium export case of PG partitions with table-list 5. Added expected files to be validated in pg-partitions automation test for table-list case of export-data-status command. --- migtests/scripts/run-test.sh | 5 ++ .../partitions/export-data-status-report.json | 37 ++++++++++ ...rt-data-status-with-table-list-report.json | 37 ++++++++++ yb-voyager/cmd/common.go | 31 +++----- yb-voyager/cmd/exportData.go | 5 -- yb-voyager/cmd/exportDataStatus.go | 7 ++ yb-voyager/cmd/exportDataStatusCommand.go | 74 ++++++++++++------- .../cmd/getDataMigrationReportCommand.go | 8 ++ 8 files changed, 152 insertions(+), 52 deletions(-) create mode 100644 migtests/tests/pg/partitions/export-data-status-report.json create mode 100644 migtests/tests/pg/partitions/export-data-status-with-table-list-report.json diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 88569b0c60..f0d9408cf8 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -168,6 +168,11 @@ main() { expected_file="${TEST_DIR}/export_data_status-report.json" actual_file="${EXPORT_DIR}/reports/export-data-status-report.json" + if [ "${EXPORT_TABLE_LIST}" != "" ] + then + expected_file="${TEST_DIR}/export-data-status-with-table-list-report.json" + fi + step "Verify export-data-status report" verify_report ${expected_file} ${actual_file} diff --git a/migtests/tests/pg/partitions/export-data-status-report.json b/migtests/tests/pg/partitions/export-data-status-report.json new file mode 100644 index 0000000000..e9e23ee1ad --- /dev/null +++ b/migtests/tests/pg/partitions/export-data-status-report.json @@ -0,0 +1,37 @@ +[ + { + "table_name": "customers", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "sales", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "emp", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "p1.sales_region", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "range_columns_partition_test", + "status": "DONE", + "exported_count": 6 + }, + { + "table_name": "sales_region", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "test_partitions_sequences", + "status": "DONE", + "exported_count": 1000 + } +] \ No newline at end of file diff --git a/migtests/tests/pg/partitions/export-data-status-with-table-list-report.json b/migtests/tests/pg/partitions/export-data-status-with-table-list-report.json new file mode 100644 index 0000000000..e76c1483aa --- /dev/null +++ b/migtests/tests/pg/partitions/export-data-status-with-table-list-report.json @@ -0,0 +1,37 @@ +[ + { + "table_name": "customers (cust_other, cust_part11, cust_part12, cust_part21, cust_part22)", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "sales (sales_2019_q4, sales_2020_q1, sales_2020_q2)", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "emp (emp_0, emp_1, emp_2)", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "p1.sales_region (p2.boston, p2.london, p2.sydney)", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "range_columns_partition_test (range_columns_partition_test_p0, range_columns_partition_test_p1)", + "status": "DONE", + "exported_count": 6 + }, + { + "table_name": "sales_region (boston, london, sydney)", + "status": "DONE", + "exported_count": 1000 + }, + { + "table_name": "test_partitions_sequences (test_partitions_sequences_b, test_partitions_sequences_l, test_partitions_sequences_s)", + "status": "DONE", + "exported_count": 1000 + } +] \ No newline at end of file diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 3bce78ffd0..bf8606ea6f 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -236,11 +236,13 @@ func getLeafPartitionsFromRootTable() map[string][]string { if err != nil { utils.ErrExit("get migration status record: %v", err) } - if !msr.IsExportTableListSet || msr.SourceDBConf.DBType != POSTGRESQL { + if msr.SourceDBConf.DBType != POSTGRESQL { return leafPartitions } tables := msr.TableListExportedFromSource for leaf, root := range msr.SourceRenameTablesMap { + //Using the SQLName here to avoid creating the NameTuples manually for leafTable case as in a case partition names changes on target + //NameRegistry won't be able to figure out the map of source->target tuples. leafTable := sqlname.NewSourceNameFromQualifiedName(getQuotedFromUnquoted(leaf)) rootTable := sqlname.NewSourceNameFromQualifiedName(getQuotedFromUnquoted(root)) leaf = leafTable.Qualified.MinQuoted @@ -251,6 +253,7 @@ func getLeafPartitionsFromRootTable() map[string][]string { if !lo.Contains(tables, root) { continue } + //Adding a Qualified.MinQuoted to key and values which is similar to NameTuple.ForOutput(); leafPartitions[root] = append(leafPartitions[root], leaf) } @@ -268,6 +271,10 @@ func displayExportedRowCountSnapshot(snapshotViaDebezium bool) { fmt.Printf("snapshot export report\n") uitable := uitable.New() + msr, err := metaDB.GetMigrationStatusRecord() + if err != nil { + utils.ErrExit("error getting migration status record: %v", err) + } leafPartitions := getLeafPartitionsFromRootTable() if !snapshotViaDebezium { exportedRowCount := getExportedRowCountSnapshot(exportDir) @@ -287,8 +294,9 @@ func displayExportedRowCountSnapshot(snapshotViaDebezium bool) { utils.ErrExit("lookup table %s in name registry : %v", key, err) } displayTableName := table.CurrentName.Unqualified.MinQuoted + //Using the ForOutput() as a key for leafPartitions map as we are populating the map in that way. partitions := leafPartitions[table.ForOutput()] - if source.DBType == POSTGRESQL && partitions != nil { + if source.DBType == POSTGRESQL && partitions != nil && msr.IsExportTableListSet { partitions := strings.Join(partitions, ", ") displayTableName = fmt.Sprintf("%s (%s)", table.CurrentName.Unqualified.MinQuoted, partitions) } @@ -353,21 +361,6 @@ func renameDatafileDescriptor(exportDir string) { datafileDescriptor.Save() } -func renameExportSnapshotStatus(exportSnapshotStatusFile *jsonfile.JsonFile[ExportSnapshotStatus]) error { - err := exportSnapshotStatusFile.Update(func(exportSnapshotStatus *ExportSnapshotStatus) { - for i, tableStatus := range exportSnapshotStatus.Tables { - renamedTable, isRenamed := renameTableIfRequired(tableStatus.TableName) - if isRenamed { - exportSnapshotStatus.Tables[i].TableName = renamedTable - } - } - }) - if err != nil { - return fmt.Errorf("update export snapshot status: %w", err) - } - return nil -} - func displayImportedRowCountSnapshot(state *ImportDataState, tasks []*ImportFileTask) { if importerRole == IMPORT_FILE_ROLE { fmt.Printf("import report\n") @@ -898,10 +891,6 @@ func getExportedSnapshotRowsMap(exportSnapshotStatus *ExportSnapshotStatus) (*ut snapshotStatusMap := utils.NewStructMap[sqlname.NameTuple, []string]() for _, tableStatus := range exportSnapshotStatus.Tables { - if tableStatus.FileName == "" { - //in case of root table as well in the tablelist during export an entry with empty file name is there - continue - } nt, err := namereg.NameReg.LookupTableName(tableStatus.TableName) if err != nil { return nil, nil, fmt.Errorf("lookup table [%s] from name registry: %v", tableStatus.TableName, err) diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index ed50cd63ff..46522c0342 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -900,11 +900,6 @@ func exportDataOffline(ctx context.Context, cancel context.CancelFunc, finalTabl if source.DBType == POSTGRESQL { //Make leaf partitions data files entry under the name of root table renameDatafileDescriptor(exportDir) - //Similarly for the export snapshot status file - err = renameExportSnapshotStatus(exportSnapshotStatusFile) - if err != nil { - return fmt.Errorf("rename export snapshot status: %w", err) - } } displayExportedRowCountSnapshot(false) diff --git a/yb-voyager/cmd/exportDataStatus.go b/yb-voyager/cmd/exportDataStatus.go index 1c2a976a51..ee994bf0e0 100644 --- a/yb-voyager/cmd/exportDataStatus.go +++ b/yb-voyager/cmd/exportDataStatus.go @@ -101,6 +101,13 @@ func initializeExportTableMetadata(tableList []sqlname.NameTuple) { Status: utils.TableMetadataStatusMap[tablesProgressMetadata[key].Status], ExportedRowCountSnapshot: int64(0), } + if source.DBType == POSTGRESQL { + //for Postgresql rename the table leaf table names to root table + renamedTable, isRenamed := renameTableIfRequired(key) + if isRenamed { + exportSnapshotStatus.Tables[key].TableName = renamedTable + } + } } err := exportSnapshotStatusFile.Create(exportSnapshotStatus) if err != nil { diff --git a/yb-voyager/cmd/exportDataStatusCommand.go b/yb-voyager/cmd/exportDataStatusCommand.go index 5f1c579801..af948a0cf4 100644 --- a/yb-voyager/cmd/exportDataStatusCommand.go +++ b/yb-voyager/cmd/exportDataStatusCommand.go @@ -20,6 +20,7 @@ import ( "fmt" "io/fs" "path/filepath" + "slices" "sort" "strings" @@ -28,6 +29,7 @@ import ( "github.com/spf13/cobra" "github.com/yugabyte/yb-voyager/yb-voyager/src/dbzm" + "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/namereg" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/jsonfile" @@ -61,12 +63,22 @@ var exportDataStatusCmd = &cobra.Command{ if err != nil { utils.ErrExit("Failed to get migration status record: %s", err) } + if msr == nil { + color.Cyan(exportDataStatusMsg) + return + } useDebezium = msr.IsSnapshotExportedViaDebezium() + + + source = *msr.SourceDBConf + sqlname.SourceDBType = source.DBType + leafPartitions := getLeafPartitionsFromRootTable() + var rows []*exportTableMigStatusOutputRow if useDebezium { - rows, err = runExportDataStatusCmdDbzm(streamChanges) + rows, err = runExportDataStatusCmdDbzm(streamChanges, leafPartitions, msr) } else { - rows, err = runExportDataStatusCmd() + rows, err = runExportDataStatusCmd(msr, leafPartitions) } if err != nil { utils.ErrExit("error: %s\n", err) @@ -105,33 +117,35 @@ var InProgressTableSno int // Note that the `export data status` is running in a separate process. It won't have access to the in-memory state // held in the main `export data` process. -func runExportDataStatusCmdDbzm(streamChanges bool) ([]*exportTableMigStatusOutputRow, error) { +func runExportDataStatusCmdDbzm(streamChanges bool, leafPartitions map[string][]string, msr *metadb.MigrationStatusRecord) ([]*exportTableMigStatusOutputRow, error) { exportStatusFilePath := filepath.Join(exportDir, "data", "export_status.json") status, err := dbzm.ReadExportStatus(exportStatusFilePath) if err != nil { utils.ErrExit("Failed to read export status file %s: %v", exportStatusFilePath, err) } if status == nil { - return nil, nil + return nil, fmt.Errorf("export data has not started yet. Try running after export has started") } InProgressTableSno = status.InProgressTableSno() var rows []*exportTableMigStatusOutputRow var row *exportTableMigStatusOutputRow for _, tableStatus := range status.Tables { - row = getSnapshotExportStatusRow(&tableStatus) + row = getSnapshotExportStatusRow(&tableStatus, leafPartitions, msr) rows = append(rows, row) } return rows, nil } -func getSnapshotExportStatusRow(tableStatus *dbzm.TableExportStatus) *exportTableMigStatusOutputRow { +func getSnapshotExportStatusRow(tableStatus *dbzm.TableExportStatus, leafPartitions map[string][]string, msr *metadb.MigrationStatusRecord) *exportTableMigStatusOutputRow { nt, err := namereg.NameReg.LookupTableName(fmt.Sprintf("%s.%s", tableStatus.SchemaName, tableStatus.TableName)) if err != nil { utils.ErrExit("lookup %s in name registry: %v", tableStatus.TableName, err) } + //Using the ForOutput() as a key for leafPartitions map as we are populating the map in that way. + displayTableName := getDisplayName(nt, leafPartitions[nt.ForOutput()], msr.IsExportTableListSet) row := &exportTableMigStatusOutputRow{ - TableName: nt.ForMinOutput(), + TableName: displayTableName, Status: "DONE", ExportedCount: tableStatus.ExportedRowCountSnapshot, } @@ -145,21 +159,27 @@ func getSnapshotExportStatusRow(tableStatus *dbzm.TableExportStatus) *exportTabl return row } -func runExportDataStatusCmd() ([]*exportTableMigStatusOutputRow, error) { - msr, err := metaDB.GetMigrationStatusRecord() - if err != nil { - return nil, fmt.Errorf("error while getting migration status record: %v", err) +func getDisplayName(nt sqlname.NameTuple, partitions []string, isTableListSet bool) string { + displayTableName := nt.ForMinOutput() + //Changing the display of the partition tables in case table-list is set because there can be case where user has passed a subset of leaft tables in the list + if source.DBType == POSTGRESQL && partitions != nil && isTableListSet { + slices.Sort(partitions) + partitions := strings.Join(partitions, ", ") + displayTableName = fmt.Sprintf("%s (%s)", displayTableName, partitions) } + + return displayTableName +} + +func runExportDataStatusCmd(msr *metadb.MigrationStatusRecord, leafPartitions map[string][]string) ([]*exportTableMigStatusOutputRow, error) { tableList := msr.TableListExportedFromSource - source = *msr.SourceDBConf - sqlname.SourceDBType = source.DBType var outputRows []*exportTableMigStatusOutputRow exportSnapshotStatusFilePath := filepath.Join(exportDir, "metainfo", "export_snapshot_status.json") exportSnapshotStatusFile = jsonfile.NewJsonFile[ExportSnapshotStatus](exportSnapshotStatusFilePath) exportStatusSnapshot, err := exportSnapshotStatusFile.Read() if err != nil { if errors.Is(err, fs.ErrNotExist) { - return nil, nil + return nil, fmt.Errorf("export data has not started yet. Try running after export has started") } utils.ErrExit("Failed to read export status file %s: %v", exportSnapshotStatusFilePath, err) } @@ -169,20 +189,17 @@ func runExportDataStatusCmd() ([]*exportTableMigStatusOutputRow, error) { return nil, fmt.Errorf("error while getting exported snapshot rows map: %v", err) } - leafPartitions := getLeafPartitionsFromRootTable() - for _, tableName := range tableList { finalFullTableName, err := namereg.NameReg.LookupTableName(tableName) if err != nil { return nil, fmt.Errorf("lookup %s in name registry: %v", tableName, err) } - displayTableName := finalFullTableName.ForMinOutput() - partitions := leafPartitions[finalFullTableName.ForOutput()] - if source.DBType == POSTGRESQL && partitions != nil { - partitions := strings.Join(partitions, ", ") - displayTableName = fmt.Sprintf("%s (%s)", displayTableName, partitions) + //Using the ForOutput() as a key for leafPartitions map as we are populating the map in that way. + displayTableName := getDisplayName(finalFullTableName, leafPartitions[finalFullTableName.ForOutput()], msr.IsExportTableListSet) + snapshotStatus, ok := exportedSnapshotStatus.Get(finalFullTableName) + if !ok { + return nil, fmt.Errorf("snapshot status for table %s is not populated in %q file", finalFullTableName.ForMinOutput(), exportSnapshotStatusFilePath) } - snapshotStatus, _ := exportedSnapshotStatus.Get(finalFullTableName) finalStatus := snapshotStatus[0] if len(snapshotStatus) > 1 { // status for root partition wrt leaf partitions exportingLeaf := 0 @@ -199,13 +216,18 @@ func runExportDataStatusCmd() ([]*exportTableMigStatusOutputRow, error) { } if exportingLeaf > 0 { finalStatus = "EXPORTING" - } else if doneLeaf == len(snapshotStatus) { - finalStatus = "DONE" } else if not_started == len(snapshotStatus) { - finalStatus = "NOT_STARTED" + //In case of partition tables in PG, we are clubbing the status of all leafs and then returning the status + //For root table we are sending NOT_STARTED and if only all leaf partitions will have NOT_STARTED else EXPORTING/DONE + finalStatus = "NOT-STARTED" + } else { + finalStatus = "DONE" } } - exportedCount, _ := exportedSnapshotRow.Get(finalFullTableName) + exportedCount, ok := exportedSnapshotRow.Get(finalFullTableName) + if !ok { + return nil, fmt.Errorf("snapshot row count for table %s is not populated in %q file", finalFullTableName.ForMinOutput(), exportSnapshotStatusFilePath) + } row := &exportTableMigStatusOutputRow{ TableName: displayTableName, Status: finalStatus, diff --git a/yb-voyager/cmd/getDataMigrationReportCommand.go b/yb-voyager/cmd/getDataMigrationReportCommand.go index 688ee8e68a..9a1a734cea 100644 --- a/yb-voyager/cmd/getDataMigrationReportCommand.go +++ b/yb-voyager/cmd/getDataMigrationReportCommand.go @@ -16,7 +16,9 @@ limitations under the License. package cmd import ( + "errors" "fmt" + "io/fs" "path/filepath" "github.com/fatih/color" @@ -137,6 +139,9 @@ func getDataMigrationReportCmdFn(msr *metadb.MigrationStatusRecord) { if err != nil { utils.ErrExit("Failed to read export status file %s: %v", exportStatusFilePath, err) } + if dbzmStatus == nil { + utils.ErrExit("Export data has not started yet. Try running after export has started.") + } dbzmNameTupToRowCount := utils.NewStructMap[sqlname.NameTuple, int64]() exportSnapshotStatusFilePath := filepath.Join(exportDir, "metainfo", "export_snapshot_status.json") @@ -150,6 +155,9 @@ func getDataMigrationReportCmdFn(msr *metadb.MigrationStatusRecord) { if source.DBType == POSTGRESQL { exportSnapshotStatus, err = exportSnapshotStatusFile.Read() if err != nil { + if errors.Is(err, fs.ErrNotExist) { + utils.ErrExit("Export data has not started yet. Try running after export has started.") + } utils.ErrExit("Failed to read export status file %s: %v", exportSnapshotStatusFilePath, err) } exportedPGSnapshotRowsMap, _, err = getExportedSnapshotRowsMap(exportSnapshotStatus) From 85d0e8aa62c5235f9137c25c8aad67c99ab026b9 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Tue, 19 Nov 2024 17:37:37 +0530 Subject: [PATCH 002/105] Support for reporting of Unsupported PLpg/SQL objects in assessment and analyze (#1897) 1. Implement the framework to parse the PLPGSQL code blocks of objects like FUNCTION/PROCEDURE to extract the individual queries / statements and then the issues of those queries and report them in analyze/assess report as Unsupported Plpg/SQL objects. https://yugabyte.atlassian.net/browse/DB-13982 2. Reporting the Unsupported Query constructs like "Advisory Locks", "System columns", and "XML functions" https://yugabyte.atlassian.net/browse/DB-14041 3. Added an ENV var to control the reporting of this feature with REPORT_UNSUPPORTED_PLPGSQL_OBJECTS. 4. Enabled this feature for analyze-schema in this PR. 5. Added tests for it in analyze-schema tests. --- .../schema/functions/function.sql | 90 +++++++ .../schema/procedures/procedure.sql | 37 +++ .../tests/analyze-schema/expected_issues.json | 120 +++++++++ migtests/tests/analyze-schema/summary.json | 10 +- yb-voyager/cmd/analyzeSchema.go | 43 +++- yb-voyager/cmd/assessMigrationCommand.go | 69 +++++- yb-voyager/cmd/common.go | 4 +- yb-voyager/cmd/constants.go | 1 + .../migration_assessment_report.template | 71 +++++- .../cmd/templates/schema_analysis_report.html | 12 +- yb-voyager/src/queryissue/queryissue.go | 28 ++- .../queryissue/unsupported_dml_constructs.go | 18 +- .../unsupported_dml_constructs_test.go | 18 +- yb-voyager/src/queryparser/helpers.go | 15 ++ yb-voyager/src/queryparser/query_parser.go | 69 ++++++ yb-voyager/src/queryparser/traversal.go | 15 ++ .../src/queryparser/traversal_plpgsql.go | 230 ++++++++++++++++++ 17 files changed, 834 insertions(+), 16 deletions(-) create mode 100644 migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql create mode 100644 yb-voyager/src/queryparser/traversal_plpgsql.go diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql new file mode 100644 index 0000000000..f85ea54d71 --- /dev/null +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql @@ -0,0 +1,90 @@ +CREATE OR REPLACE FUNCTION create_and_populate_tables(table_prefix TEXT, partitions INT) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +DECLARE + i INT; + partition_table TEXT; +BEGIN + -- Loop to create multiple partition tables + FOR i IN 1..partitions LOOP + partition_table := table_prefix || '_part_' || i; + + -- Dynamic SQL to create each partition table + EXECUTE format(' + CREATE TABLE IF NOT EXISTS %I ( + id SERIAL PRIMARY KEY, + name TEXT, + amount NUMERIC + )', partition_table); + RAISE NOTICE 'Table % created', partition_table; + + -- Dynamic SQL to insert data into each partition table + EXECUTE format(' + INSERT INTO %I (name, amount) + SELECT name, amount + FROM source_data + WHERE id %% %L = %L', partition_table, partitions, i - 1); + RAISE NOTICE 'Data inserted into table %', partition_table; + END LOOP; + + PERFORM pg_advisory_lock(sender_id); + PERFORM pg_advisory_lock(receiver_id); + + -- Check if the sender has enough balance + IF (SELECT balance FROM accounts WHERE account_id = sender_id) < transfer_amount THEN + RAISE EXCEPTION 'Insufficient funds'; + -- Deduct the amount from the sender's account SOME DUMMY code to understand nested if structure + UPDATE accounts + SET balance = balance - transfer_amount + WHERE account_id = sender_id; + + -- Add the amount to the receiver's account + UPDATE accounts + SET balance = balance + transfer_amount + WHERE account_id = receiver_id; + IF (SELECT balance FROM accounts WHERE account_id = sender_id) < transfer_amount THEN + -- Release the advisory locks (optional, as they will be released at the end of the transaction) + PERFORM pg_advisory_unlock(sender_id); + PERFORM pg_advisory_unlock(receiver_id); + END IF; + END IF; + + -- Deduct the amount from the sender's account + UPDATE accounts + SET balance = balance - transfer_amount + WHERE account_id = sender_id; + + -- Add the amount to the receiver's account + UPDATE accounts + SET balance = balance + transfer_amount + WHERE account_id = receiver_id; + + -- Commit the transaction + COMMIT; + + -- Release the advisory locks (optional, as they will be released at the end of the transaction) + PERFORM pg_advisory_unlock(sender_id); + PERFORM pg_advisory_unlock(receiver_id); + + -- Conditional logic + IF balance >= withdrawal THEN + RAISE NOTICE 'Sufficient balance, processing withdrawal.'; + -- Add the amount to the receiver's account + UPDATE accounts SET balance = balance + amount WHERE account_id = receiver; + ELSIF balance > 0 AND balance < withdrawal THEN + RAISE NOTICE 'Insufficient balance, consider reducing the amount.'; + -- Add the amount to the receiver's account + UPDATE accounts SET balance = balance + amount WHERE account_id = receiver; + ELSE + -- Add the amount to the receiver's account + UPDATE accounts SET balance = balance + amount WHERE account_id = receiver; + RAISE NOTICE 'No funds available.'; + END IF; + + SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type; + + SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department); + +END; +$$; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql index 900f71e8b4..4879281eb3 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql @@ -112,3 +112,40 @@ $body$ LANGUAGE PLPGSQL SECURITY DEFINER ; + +CREATE OR REPLACE PROCEDURE add_employee( + emp_name VARCHAR, + emp_age INT +) +LANGUAGE plpgsql +AS $$ +BEGIN + SELECT id, first_name FROM employees WHERE pg_try_advisory_lock(300) IS TRUE; + + -- Insert a new record into the employees table + INSERT INTO employees(name, age) + VALUES (emp_name, emp_age); + + SELECT e.id, e.name, + ROW_NUMBER() OVER (ORDER BY e.ctid) AS row_num + FROM employees e; + + SELECT e.id, x.employee_xml + FROM employees e + JOIN ( + SELECT xmlelement(name "employee", xmlattributes(e.id AS "id"), e.name) AS employee_xml + FROM employees e + ) x ON x.employee_xml IS NOT NULL + WHERE xmlexists('//employee[name="John Doe"]' PASSING BY REF x.employee_xml); + + SELECT e.id, + CASE + WHEN e.salary > 100000 THEN pg_advisory_lock(e.id) + ELSE pg_advisory_unlock(e.id) + END AS lock_status + FROM employees e; + + -- Optional: Log a message + RAISE NOTICE 'Employee % of age % added successfully.', emp_name, emp_age; +END; +$$; diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index e3eacf1d81..1ee5b52e4a 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1253,5 +1253,125 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT pg_advisory_lock(sender_id);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT pg_advisory_lock(receiver_id);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT pg_advisory_unlock(sender_id);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT pg_advisory_unlock(receiver_id);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT pg_advisory_unlock(sender_id);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT pg_advisory_unlock(receiver_id);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "XML Functions", + "SqlStatement": "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "create_and_populate_tables", + "Reason": "System Columns", + "SqlStatement": "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "add_employee", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT id, first_name FROM employees WHERE pg_try_advisory_lock(300) IS TRUE;", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "add_employee", + "Reason": "System Columns", + "SqlStatement": "SELECT e.id, e.name,\n ROW_NUMBER() OVER (ORDER BY e.ctid) AS row_num\n FROM employees e;", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "add_employee", + "Reason": "XML Functions", + "SqlStatement": "SELECT e.id, x.employee_xml\n FROM employees e\n JOIN (\n SELECT xmlelement(name \"employee\", xmlattributes(e.id AS \"id\"), e.name) AS employee_xml\n FROM employees e\n ) x ON x.employee_xml IS NOT NULL\n WHERE xmlexists('//employee[name=\"John Doe\"]' PASSING BY REF x.employee_xml);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "add_employee", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT e.id,\n CASE\n WHEN e.salary \u003e 100000 THEN pg_advisory_lock(e.id)\n ELSE pg_advisory_unlock(e.id)\n END AS lock_status\n FROM employees e;", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 18fd6e6061..4e710c672f 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -34,11 +34,17 @@ "ObjectNames": "film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." }, + { + "ObjectType": "FUNCTION", + "TotalCount": 1, + "InvalidCount": 0, + "ObjectNames": "create_and_populate_tables" + }, { "ObjectType": "PROCEDURE", - "TotalCount": 5, + "TotalCount": 6, "InvalidCount": 3, - "ObjectNames": "foo, foo1, sp_createnachabatch, test, test1" + "ObjectNames": "foo, foo1, sp_createnachabatch, test, test1, add_employee" }, { "ObjectType": "VIEW", diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 2ba76f6a03..451e32971a 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -35,7 +35,9 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryissue" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" @@ -126,10 +128,11 @@ var ( schemaAnalysisReport utils.SchemaReport partitionTablesMap = make(map[string]bool) // key is partitioned table, value is sqlInfo (sqlstmt, fpath) where the ADD PRIMARY KEY statement resides - primaryConsInAlter = make(map[string]*sqlInfo) - summaryMap = make(map[string]*summaryInfo) - multiRegex = regexp.MustCompile(`([a-zA-Z0-9_\.]+[,|;])`) - dollarQuoteRegex = regexp.MustCompile(`(\$.*\$)`) + primaryConsInAlter = make(map[string]*sqlInfo) + summaryMap = make(map[string]*summaryInfo) + parserIssueDetector = queryissue.NewParserIssueDetector() + multiRegex = regexp.MustCompile(`([a-zA-Z0-9_\.]+[,|;])`) + dollarQuoteRegex = regexp.MustCompile(`(\$.*\$)`) /* this will contain the information in this format: public.table1 -> { @@ -1579,9 +1582,39 @@ func checker(sqlInfoArr []sqlInfo, fpath string, objType string) { checkDDL(sqlInfoArr, fpath, objType) checkForeign(sqlInfoArr, fpath) checkRemaining(sqlInfoArr, fpath) + if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { + checkPlPgSQLStmtsUsingParser(sqlInfoArr, fpath, objType) + } checkStmtsUsingParser(sqlInfoArr, fpath, objType) } +func checkPlPgSQLStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { + for _, sqlInfoStmt := range sqlInfoArr { + issues, err := parserIssueDetector.GetIssues(sqlInfoStmt.formattedStmt) + if err != nil { + log.Infof("error in getting the issues-%s: %v", sqlInfoStmt.formattedStmt, err) + continue + } + for _, issueInstance := range issues { + issue := convertIssueInstanceToAnalyzeIssue(issueInstance, fpath) + schemaAnalysisReport.Issues = append(schemaAnalysisReport.Issues, issue) + } + } + +} + +func convertIssueInstanceToAnalyzeIssue(issueInstance issue.IssueInstance, fileName string) utils.Issue { + return utils.Issue{ + ObjectType: issueInstance.ObjectType, + ObjectName: issueInstance.ObjectName, + Reason: issueInstance.TypeName, + SqlStatement: issueInstance.SqlStatement, //Displaying the actual query in the PLPGSQL block that is problematic + DocsLink: issueInstance.DocsLink, + FilePath: fileName, + IssueType: UNSUPPORTED_PLPGSQL_OBEJCTS, + } +} + func checkExtensions(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { if sqlInfo.objName != "" && !slices.Contains(supportedExtensionsOnYB, sqlInfo.objName) { @@ -1705,7 +1738,7 @@ func getObjectNameWithTable(stmt string, regexObjName string) string { parsedTree, err := pg_query.Parse(stmt) if err != nil { // in case it is not able to parse stmt as its not in PG syntax so returning the regex name - log.Errorf("Erroring parsing the the stmt %s - %v", stmt, err) + log.Errorf("Error parsing the the stmt %s - %v", stmt, err) return regexObjName } var objectName *sqlname.ObjectName diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index d3e453b23a..0fd1bb8199 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -857,6 +857,10 @@ func getAssessmentReportContentFromAnalyzeSchema() error { } assessmentReport.UnsupportedFeatures = append(assessmentReport.UnsupportedFeatures, unsupportedFeatures...) assessmentReport.UnsupportedFeaturesDesc = FEATURE_ISSUE_TYPE_DESCRIPTION + if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", false) { + unsupportedPlpgSqlObjects := fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport) + assessmentReport.UnsupportedPlPgSqlObjects = unsupportedPlpgSqlObjects + } return nil } @@ -986,6 +990,41 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { return unsupportedFeatures, nil } +func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []UnsupportedFeature { + if source.DBType != POSTGRESQL { + return nil + } + analyzeIssues := schemaAnalysisReport.Issues + plpgsqlIssues := lo.Filter(analyzeIssues, func(issue utils.Issue, _ int) bool { + return issue.IssueType == UNSUPPORTED_PLPGSQL_OBEJCTS + }) + groupPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.Issue) string { + return issue.Reason + }) + var unsupportedPlpgSqlObjects []UnsupportedFeature + for reason, issues := range groupPlpgsqlIssuesByReason { + var objects []ObjectInfo + var docsLink string + for _, issue := range issues { + objects = append(objects, ObjectInfo{ + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + }) + docsLink = issue.DocsLink + } + feature := UnsupportedFeature{ + FeatureName: reason, + DisplayDDL: true, + DocsLink: docsLink, + Objects: objects, + } + unsupportedPlpgSqlObjects = append(unsupportedPlpgSqlObjects, feature) + } + return unsupportedPlpgSqlObjects + +} + func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error) { if source.DBType != POSTGRESQL { return nil, nil @@ -1249,7 +1288,11 @@ func generateAssessmentReportHtml(reportDir string) error { log.Infof("creating template for assessment report...") funcMap := template.FuncMap{ - "split": split, + "split": split, + "groupByObjectType": groupByObjectType, + "numKeysInMapStringObjectInfo": numKeysInMapStringObjectInfo, + "groupByObjectName": groupByObjectName, + "totalUniqueObjectNamesOfAllTypes": totalUniqueObjectNamesOfAllTypes, } tmpl := template.Must(template.New("report").Funcs(funcMap).Parse(string(bytesTemplate))) @@ -1267,6 +1310,30 @@ func generateAssessmentReportHtml(reportDir string) error { return nil } +func groupByObjectType(objects []ObjectInfo) map[string][]ObjectInfo { + return lo.GroupBy(objects, func(object ObjectInfo) string { + return object.ObjectType + }) +} + +func groupByObjectName(objects []ObjectInfo) map[string][]ObjectInfo { + return lo.GroupBy(objects, func(object ObjectInfo) string { + return object.ObjectName + }) +} + +func totalUniqueObjectNamesOfAllTypes(m map[string][]ObjectInfo) int { + totalObjectNames := 0 + for _, objects := range m { + totalObjectNames += len(lo.Keys(groupByObjectName(objects))) + } + return totalObjectNames +} + +func numKeysInMapStringObjectInfo(m map[string][]ObjectInfo) int { + return len(lo.Keys(m)) +} + func split(value string, delimiter string) []string { return strings.Split(value, delimiter) } diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index bf8606ea6f..6037158f9c 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1124,8 +1124,9 @@ type AssessmentReport struct { UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` - MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` + UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects,omitempty"` + MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` Notes []string `json:"Notes"` } @@ -1139,6 +1140,7 @@ type UnsupportedFeature struct { } type ObjectInfo struct { + ObjectType string `json:"-"` //only using as of now for html report Unsupported PLpg/SQL objects feature ObjectName string SqlStatement string } diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 1470c873da..9eeaa4895a 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -101,6 +101,7 @@ const ( UNSUPPORTED_FEATURES = "unsupported_features" UNSUPPORTED_DATATYPES = "unsupported_datatypes" + UNSUPPORTED_PLPGSQL_OBEJCTS = "unsupported_plpgsql_objects" REPORT_UNSUPPORTED_QUERY_CONSTRUCTS = "REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" HTML = "html" diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 5e69811908..bd949d964b 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -35,7 +35,7 @@ th { background-color: #f2f2f2; } - tr:nth-child(even){background-color: #f9f9f9;} + .formatted_table tr:nth-child(even){background-color: #f9f9f9;} ul { padding-left: 20px; } @@ -59,6 +59,16 @@ .list_item { margin-bottom: 15px; } + pre { + width: 100%; /* Ensure the pre/code content takes full width of the container */ + word-wrap: break-word; /* Break long lines into multiple lines */ + overflow-wrap: break-word; /* Same as word-wrap but for newer browsers */ + white-space: pre-wrap; /* Preserve whitespace and allow wrapping */ + word-break: break-all; /* Prevents long words from overflowing */ + margin: 0; /* Remove default margins */ + padding: 0; /* Remove default padding */ + font-family: monospace; /* Optional: ensure a monospaced font */ + } @@ -159,7 +169,7 @@

{{.UnsupportedDataTypesDesc}}

{{ if .UnsupportedDataTypes }}
- +
@@ -230,7 +240,7 @@

Unsupported Query Constructs

{{ if .UnsupportedQueryConstructs}}

Source database queries not supported in YugabyteDB, identified by scanning system tables:

-
Schema Table
+
@@ -282,6 +292,61 @@

No unsupported query constructs found in the source database for target YugabyteDB.

{{ end }} +

Unsupported PL/pgSQL objects

+ {{ if .UnsupportedPlPgSqlObjects}} +

Source schema objects having unsupported statements in PLPGSQL code block:

+
Construct Type Queries
+ + + + + + + + {{ range .UnsupportedPlPgSqlObjects }} + + + {{ $objectsGroupByObjectType := groupByObjectType .Objects}} + {{ $numUniqueObjectNamesOfAllTypes := totalUniqueObjectNamesOfAllTypes $objectsGroupByObjectType}} + {{ $docsLink := .DocsLink }} + + {{ $isNextRowRequiredForObjectType := false }} + {{ range $type, $objectsByType := $objectsGroupByObjectType }} + {{ $objectGroupByObjectName := groupByObjectName $objectsByType }} + {{ $numUniqueObjectNames := numKeysInMapStringObjectInfo $objectGroupByObjectName }} + {{ if $isNextRowRequiredForObjectType }} + + {{ end }} + + {{ $isNextRowRequiredForObjectName := false }} + {{ range $name, $objectsByName := $objectGroupByObjectName }} + {{ if $isNextRowRequiredForObjectName }} + + {{ end }} +   + + {{ if not $isNextRowRequiredForObjectType }} + + {{ end }} + {{ $isNextRowRequiredForObjectName = true }} + {{ $isNextRowRequiredForObjectType = true }} + + {{ end }} + {{ end }} + {{ end }} +
Feature NameObject typeObject nameStatementDetails
{{ .FeatureName }}
{{$type}}
{{ $name }} +
+
    + {{ range $objectsByName }} + 
  • {{ .SqlStatement }}
    • + {{ end }} +
+
+ 		Link
+ {{ else }} +

No unsupported PL/pgSQL objects found in the source database for target YugabyteDB.

+ {{ end }} + {{ if .MigrationCaveats}} {{ $hasMigrationCaveats := false }}

Migration caveats

diff --git a/yb-voyager/cmd/templates/schema_analysis_report.html b/yb-voyager/cmd/templates/schema_analysis_report.html index 8d65af904e..00ee63d3be 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.html +++ b/yb-voyager/cmd/templates/schema_analysis_report.html @@ -44,6 +44,16 @@ font-weight: bold; background-color: #f9f9f9; } + pre { + width: 100%; /* Ensure the pre/code content takes full width of the container */ + word-wrap: break-word; /* Break long lines into multiple lines */ + overflow-wrap: break-word; /* Same as word-wrap but for newer browsers */ + white-space: pre-wrap; /* Preserve whitespace and allow wrapping */ + word-break: break-all; /* Prevents long words from overflowing */ + margin: 0; /* Remove default margins */ + padding: 0; /* Remove default padding */ + font-family: Arial, Helvetica, sans-serif; /* Optional: ensure a monospaced font */ + } @@ -109,7 +119,7 @@

Issue in Object {{ $issue.ObjectType }}

  • Issue Type: {{ $issue.IssueType }}
  • Object Name: {{ $issue.ObjectName }}
  • Reason: {{ $issue.Reason }}
    • -
  • SQL Statement: {{ $issue.SqlStatement }}
    • +
  • SQL Statement: 
    {{ $issue.SqlStatement }}
  • File Path: {{ $issue.FilePath }} [Preview]
    • {{ if $issue.Suggestion }}
  • Suggestion: {{ $issue.Suggestion }}
    • diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 7154fffeb6..8b12140f4c 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -22,9 +22,10 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v5" "github.com/samber/lo" log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/reflect/protoreflect" + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" - "google.golang.org/protobuf/reflect/protoreflect" ) type ParserIssueDetector struct { @@ -41,6 +42,31 @@ func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, er if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) } + if queryparser.IsPLPGSQLObject(parseTree) { + plpgsqlObjType, plpgsqlObjName := queryparser.GetObjectTypeAndObjectName(parseTree) + plpgsqlQueries, err := queryparser.GetAllPLPGSQLStatements(query) + if err != nil { + return nil, fmt.Errorf("error getting all the queries from query: %w", err) + } + var issues []issue.IssueInstance + for _, plpgsqlQuery := range plpgsqlQueries { + issuesInQuery, err := p.GetIssues(plpgsqlQuery) + if err != nil { + //there can be plpgsql expr queries no parseable via parser e.g. "withdrawal > balance" + log.Errorf("error getting issues in query-%s: %v", query, err) + continue + } + for _, i := range issuesInQuery { + //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object + //e.g. replacing the DML_QUERY and "" to FUNCTION and + i.ObjectType = plpgsqlObjType + i.ObjectName = plpgsqlObjName + issues = append(issues, i) + } + } + return issues, nil + } + //TODO: add handling for VIEW/MVIEW to parse select return p.getDMLIssues(query, parseTree) } diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs.go b/yb-voyager/src/queryissue/unsupported_dml_constructs.go index d81206b4f4..0077e699d3 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs.go +++ b/yb-voyager/src/queryissue/unsupported_dml_constructs.go @@ -1,9 +1,25 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package queryissue import ( log "github.com/sirupsen/logrus" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" "google.golang.org/protobuf/reflect/protoreflect" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" ) const ( diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go b/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go index 88fba29493..d9974f1854 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go +++ b/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go @@ -1,3 +1,18 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package queryissue import ( @@ -9,8 +24,9 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" "google.golang.org/protobuf/reflect/protoreflect" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" ) // TestFuncCallDetector tests the Advisory Lock Detector. diff --git a/yb-voyager/src/queryparser/helpers.go b/yb-voyager/src/queryparser/helpers.go index 9eca161b04..7b57ddbbb0 100644 --- a/yb-voyager/src/queryparser/helpers.go +++ b/yb-voyager/src/queryparser/helpers.go @@ -1,3 +1,18 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package queryparser import "google.golang.org/protobuf/reflect/protoreflect" diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/queryparser/query_parser.go index f90e6faabc..1b735814b5 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/queryparser/query_parser.go @@ -1,7 +1,25 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package queryparser import ( + "fmt" + pg_query "github.com/pganalyze/pg_query_go/v5" + "github.com/samber/lo" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -15,6 +33,57 @@ func Parse(query string) (*pg_query.ParseResult, error) { return tree, nil } +func ParsePLPGSQLToJson(query string) (string, error) { + log.Debugf("parsing the PLPGSQL to json query-%s", query) + jsonString, err := pg_query.ParsePlPgSqlToJSON(query) + if err != nil { + return "", err + } + return jsonString, err +} + func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { return parseTree.Stmts[0].Stmt.ProtoReflect() } + +func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { + // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE + _, isPlPgSQLObject := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) + return isPlPgSQLObject +} + +func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { + createFuncNode, isCreateFunc := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) + switch true { + case isCreateFunc: + /* + version:160001 stmts:{stmt:{create_function_stmt:{replace:true funcname:{string:{sval:"public"}} funcname:{string:{sval:"add_employee"}} + parameters:{function_parameter:{name:"emp_name" arg_type:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"varchar"}} + typemod:-1 location:62} mode:FUNC_PARAM_DEFAULT}} parameters:{funct ... + + version:160001 stmts:{stmt:{create_function_stmt:{is_procedure:true replace:true funcname:{string:{sval:"public"}} + funcname:{string:{sval:"add_employee"}} parameters:{function_parameter:{name:"emp_name" arg_type:{names:{string:{sval:"pg_catalog"}} + names:{string:{sval:"varchar"}} typemod:-1 location:63} mode:FUNC_PARAM_DEFAULT}} ... + */ + stmt := createFuncNode.CreateFunctionStmt + objectType := "FUNCTION" + if stmt.IsProcedure { + objectType = "PROCEDURE" + } + funcNameList := stmt.GetFuncname() + return objectType, getFunctionObjectName(funcNameList) + } + return "", "" +} + +func getFunctionObjectName(funcNameList []*pg_query.Node) string { + funcName := "" + funcSchemaName := "" + if len(funcNameList) > 0 { + funcName = funcNameList[len(funcNameList)-1].GetString_().Sval // func name can be qualified / unqualifed or native / non-native proper func name will always be available at last index + } + if len(funcNameList) >= 2 { // Names list will have all the parts of qualified func name + funcSchemaName = funcNameList[len(funcNameList)-2].GetString_().Sval // // func name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index + } + return lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) +} diff --git a/yb-voyager/src/queryparser/traversal.go b/yb-voyager/src/queryparser/traversal.go index 9c231bd2d2..090e30f055 100644 --- a/yb-voyager/src/queryparser/traversal.go +++ b/yb-voyager/src/queryparser/traversal.go @@ -1,3 +1,18 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package queryparser import ( diff --git a/yb-voyager/src/queryparser/traversal_plpgsql.go b/yb-voyager/src/queryparser/traversal_plpgsql.go new file mode 100644 index 0000000000..2fa757930c --- /dev/null +++ b/yb-voyager/src/queryparser/traversal_plpgsql.go @@ -0,0 +1,230 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryparser + +import ( + "encoding/json" + "fmt" + "strings" + + log "github.com/sirupsen/logrus" +) + +const ( + PLPGSQL_EXPR = "PLpgSQL_expr" + QUERY = "query" + + ACTION = "action" + PLPGSQL_FUNCTION = "PLpgSQL_function" +) + +/* +* +This function is not concrete yet because of following limitation from parser - +The following queries are not the actual query we need so if we pass all these queries directly to parser again to detect the unsupported feature/construct. It will fail for some of these with syntax error, e.g. + + a. query "balance > 0 AND balance < withdrawal;" error - syntax error at or near "balance" + b. query "format(' + CREATE TABLE IF NOT EXISTS %I ( + id SERIAL PRIMARY KEY, + name TEXT, + amount NUMERIC + )', partition_table);" error - syntax error at or near "format" + c. query "(SELECT balance FROM accounts WHERE account_id = sender_id) < transfer_amount;" error - syntax error at or near "<" + +These issues are majorly expressions, conditions, assignments, loop variables, raise parameters, etc… and the parser is giving all these as queries so we can’t differentiate as such between actual query and these. +* +*/ +func GetAllPLPGSQLStatements(query string) ([]string, error) { + parsedJson, err := ParsePLPGSQLToJson(query) + if err != nil { + log.Infof("error in parsing the stmt-%s to json: %v", query, err) + return []string{}, err + } + if parsedJson == "" { + return []string{}, nil + } + var parsedJsonMapList []map[string]interface{} + //Refer to the queryparser.traversal_plpgsql.go for example and sample parsed json + log.Debugf("parsing the json string-%s of stmt-%s", parsedJson, query) + err = json.Unmarshal([]byte(parsedJson), &parsedJsonMapList) + if err != nil { + return []string{}, fmt.Errorf("error parsing the json string of stmt-%s: %v", query, err) + } + + if len(parsedJsonMapList) == 0 { + return []string{}, nil + } + + parsedJsonMap := parsedJsonMapList[0] + + function := parsedJsonMap[PLPGSQL_FUNCTION] + parsedFunctionMap, ok := function.(map[string]interface{}) + if !ok { + return []string{}, fmt.Errorf("error getting the PlPgSQL_Function field in parsed json-%s", parsedJson) + } + + actions := parsedFunctionMap[ACTION] + var plPgSqlStatements []string + TraversePlPgSQLJson(actions, &plPgSqlStatements) + return plPgSqlStatements, nil +} + +/* +Query example- + + CREATE OR REPLACE FUNCTION func_example_advi( + sender_id INT, + receiver_id INT, + transfer_amount NUMERIC + +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + + -- Acquire advisory locks to prevent concurrent updates on the same accounts + PERFORM pg_advisory_lock(sender_id); + PERFORM pg_advisory_lock(receiver_id); + + -- Deduct the amount from the sender's account + UPDATE accounts + SET balance = balance - transfer_amount + WHERE account_id = sender_id; + + -- Add the amount to the receiver's account + UPDATE accounts + SET balance = balance + transfer_amount + WHERE account_id = receiver_id; + + -- Release the advisory locks + PERFORM pg_advisory_unlock(sender_id); + PERFORM pg_advisory_unlock(receiver_id); + +END; +$$; + +parsed json - + + { + "PLpgSQL_function": { + "datums": [ + ... + ], + "action": { + "PLpgSQL_stmt_block": { + "lineno": 2, + "body": [ + { + "PLpgSQL_stmt_perform": { + "lineno": 4, + "expr": { + "PLpgSQL_expr": { + "query": "SELECT pg_advisory_lock(sender_id)", + "parseMode": 0 + } + } + } + }, + { + ... similar to above + }, + { + "PLpgSQL_stmt_execsql": { + "lineno": 8, + "sqlstmt": { + "PLpgSQL_expr": { + "query": "UPDATE accounts \n SET balance = balance - transfer_amount \n WHERE account_id = sender_id", + "parseMode": 0 + } + } + } + }, + { + .... similar to above + }, + { + ... similar to below + }, + { + "PLpgSQL_stmt_perform": { + "lineno": 19, + "expr": { + "PLpgSQL_expr": { + "query": "SELECT pg_advisory_unlock(receiver_id)", + "parseMode": 0 + } + } + } + }, + { + "PLpgSQL_stmt_return": {} + } + ] + } + } + } + } +*/ +func TraversePlPgSQLJson(fieldValue interface{}, plPgSqlStatements *[]string) { + fieldMap, isMap := fieldValue.(map[string]interface{}) + fieldList, isList := fieldValue.([]interface{}) + switch true { + case isMap: + for k, v := range fieldMap { + switch k { + // base case of recursive calls to reach this PLPGSQL_EXPR field in json which will have "query" field with statement + case PLPGSQL_EXPR: + expr, ok := v.(map[string]interface{}) + if ok { + query, ok := expr[QUERY] + if ok { + q := formatExprQuery(query.(string)) // formating the query of parsed json if required + + *plPgSqlStatements = append(*plPgSqlStatements, q) + } + } + default: + TraversePlPgSQLJson(v, plPgSqlStatements) + } + } + case isList: + //In case the value of a field is not a but a list of e.g. "body" + for _, l := range fieldList { + TraversePlPgSQLJson(l, plPgSqlStatements) + } + } + +} + +// Function to format the PLPGSQL EXPR query from the json string +func formatExprQuery(q string) string { + /* + PLPGSQL line - + EXECUTE 'DROP TABLE IF EXISTS employees'; + + json str - + "PLpgSQL_expr": { + "query": "'DROP TABLE IF EXISTS employees'", + */ + q = strings.Trim(q, "'") //to remove any extra '' around the statement + q = strings.TrimSpace(q) + if !strings.HasSuffix(q, ";") { // adding the ; to query in case not added already + q += ";" + } + return q +} From 91d3e8411dbdd448ea1c7c11a5033ddc37846fd7 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 20 Nov 2024 14:48:14 +0530 Subject: [PATCH 003/105] [DB-13958] Assess Migration fails if pg_stat_statements is created in a non public schema (#1912) - now metadata script checks if pgss ext was created in the provided schema_list + public. - if yes, reports unsupported query constructs - if no, it will skip reporting - Updated gather-assessment-metadata.tar.gz --- .../data/gather-assessment-metadata.tar.gz | Bin 9980 -> 11086 bytes .../postgresql/db-queries-summary.psql | 2 +- ...b-voyager-pg-gather-assessment-metadata.sh | 103 +++++++++++++----- 3 files changed, 74 insertions(+), 31 deletions(-) diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz b/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz index 166bfb4887bcebe86db7386360f25690830bcfb2..ea97be02df8724ce7e2e6e1faea0b8f262c85904 100644 GIT binary patch literal 11086 zcmV-UE3wociwFP!000001MEF(ciXm-`MUZQsN$4TuBnHg=QvTeicKflI+82NO|!0# zE=AEYHx$Vbq#fTT``dSB0Fof}u$?AOd)GWCwMYVk!MtYxuv0U-utVMC+~#~{dyzh~ zBXeR#X8qCEx*Blp?r!7X=I(Yw{hM5mnp-=Y8ymYjjmE~KMss_2WAhQ){%RPkxMCie zA!Co?aqLC$igq`=|C?UZ<@^tP9!*1=e{lb#DBO*R=qi-*OCO@XpTGQvPpk6#2iky$$?teAAiB>;F0b_xnbB zXt1I2=9mrJFOLis*|T62nPb;x?Llc^92xt=5`zmR;M$q2XE&w8e(w$4?s4nFo|z-h zoY}QtIs&3b^dB2#ikIHLHu?s&oQ$0bYq5#|%zU^*C=5EG)QwGUGiuG?7QsPdD-J^d zlQyVStItZM-z`75R?&@orY(?Ei(hK&ZD;tJ?GJusuZ{MB(TDB_{oeePpM7tB-1l`m z{U_S!jyJJC>fHI4&F`ZAH|hUIb7!0MzuDYsZf)(N{%=C_2mQZ?OV`JX?x8;ISU{>kBdY$f~_sAt4n3l~v*6RYBeKPDR zL+M@V$T%D_1hD`DQHKQ%b`P8y-9l#x2Enyqo|Pg)di^5htHQqsES2_vcm-r+9l5`b zeb=$|I+DFwCE(rOaIS-FCK?F593F%YI+H35fiE(#Zo&P#O1s~0zdPko=y=mn2)T=t-*Pc;|k_^4)mmF`{IZT{+Q( zR33U%3GEND6OxEA3w@@4&Ss#RPSR@TOhe*+702quQ4mK2-SWK%eKPjI`vqEWlBYa( zTzi3+9rmQU<@~{ell^;Mck%y1Bsu&AS9pgA;3ogS+1x1l|J{v^2mgN`*MlSfOFD9- z*Rs(HUJZO14V^Y8Cn(da{oYA;Sb17ab1X9g4>gXXB+0`2IhP9$y?Dl6utw@6;gQ29 zp&tbHq`DHbocqMC#&4a$aKI`#1SNjW6fqGD2&9%o0;(fUfqkFJOTGzD)T#tXKj~%Z*Hu**AF*+Vl&Cr=%L|_`K z>|H!HF-tsnxz0MCRVzKFMn{fidmQ{*48|y=ETRoY4F1SB>+DxMVpEQ55Xe>sMD-HIopo`QfzgiiW&?)Y&(>149R-={n>7d30rlHo*ZaDqW9Cv;;$#x!M58 zW_qj){-rZ0vzP5bXRuCC&=mGghU_gG-0rYr3@{nq?{yD4!%nXYcZX~Q9AWGn|1 zCj`fJW==$zCZC^LtyJQ6#B@j`2mAfb@o;p|>9^J@%$hJrOUv+U;)LW%==Kko?d#E? zchcWC_Vl&J=Q697*^e(+ePUnMz1Vfxk3X{DY65;gf!J-oF#seMh6&JGqgpDxHja)* zL*ut$t6VzaFpE95rnG0z(MkKB2VoagNXU1+V9yno1$UW=8zm-g1o*xvmG*ttg}$5} z3oAMJ<%p&AYb<;N0LLOKB^f!U_2G;tRV#J<2v8aVrTh?SCS}$Pr!iH`Q6?Nr%p=o_ z2+9mB6&oX;T-U!sE|%I!>%EeQI*JoYA|eIJtPPWGKxlDjf!tm$lA61;%qf5#f!-4D ze!76C_+t+du207JTU!l@86J;AsrLycoCOeN*`a&QKE!?`l*+us(2QT$z(3=R8+5o{t7fzCYb>UbSImU$)myEGsLaauL-!P+( zCMX!NRSV)V*oaTu1ncq1@h*W!et4}Ro=ZfE0H8R)Zx{?(vjE&I;AP7KY}Sz_9lBbz@8u^)%vA1WZl z+=gK$yjo+G1N+>J-H0_e8r7B1N~MHr(CqltaeFX$+v^{+o(DkhE6~jsdAl*M06)3D zWxA*Vd)r%^8}-?BVh6OmAi!-Md0LPBddAFpG8uTi(*O}Uc5SpFwp0%voKN6lhJfUu zUl+a$CW8x~Xj)MWlmu#mCNLg5Qft7!g6;P%ozVAioLZ?|0(-rGVO#H0Td(V&RRQw@ z)TR=!`xI<&v&>p8Ru=Ed?Ccpf^@xhLb>Tyc)VP+#(~lq}jncW34FI4_5U|v5W_Atr z3Y7fPnb?zL#K&c(1G6?4jFs5##-Fs<>~x{bo+&eEmLRLj5Sm78>w6<4NIC!yI*|g6 zol^ZNEd(!iNuaF<2tdZgmPBfDnFz3m%&oO%NuhIVZKJe=(5%|5_hbNhKRaC`$Y*R#f+(|3kiJ6~6NsD4q0E#}uk;z158RGMK!MTM zLNy&c)DeMZGQj6D%4C2ov@5)mb1I5N64LkBYfF8#DMn+OXBI_F^B&`e?* z5ed3j92C&Vc-tuKkmbxoFpWB5CqV?(*GKXlb;cbM@hnK5l?WNB4+YA=LqQjMrWj>H zJq@-@7sPFn##&;L3EsIFm4b-InB>q)F0>xh;gqcb0Q3;QYfpdI*x4ET^eOka{PcHF z3RNo<;zJ|J3>U=xG4nA;xpFXNG$)f}bTKAS(vMO0Q=|*fGz6y!c%TWyGi(fo5AcU5 zF%CVEw26f;K}(1f{z-&iOP*}J__4WA8vn_zfP+&z5+-ZvVVFgOV6vJ>2N@SmV;pym zjlr-zJQlR1yMwe^`eSCbpb5esh~i^GA5>S-R9*m0)NZN1 zNR)`+3R;yQmq}DrJgIBvaM02|X$>0uE*d;a`6DnW)$SwTijZgh@9-#RxzK?Ko2 zmeUjxb7U3b=}%{$%g;bpQ*sL>O$HIMoPfMQD~Yk{uu&M>xwm7O%KP*QFcAMte9tba zCP9UPBNF!W!>O46qg6@pswTprn(6#BpTaazc4puLQ|3NqeVhC4rA&~<@EeQ!lsNfI zeLZzu8ZA_;mCjyN9|Xb@hXL44$Q@A>mN$PI0okI#Hjr+e4~wcmw&33LLYCm9N1V-+wxjzZIVFP3Kb$oGJ= z=utsb7u8AhsQtlWWw~&~rNK|ujpRL-%SLkLLsLi4o=GW|QFCGRGW!A45dYp_R^PV! zU10xPlL~Hoxmlpqqma?Vm}8O@Zt}$)O0z|gshnq3xf5whDYF=mBZGJy%a`DJ17N5K z!l@=AHj7ZCQ>O(eo7dO@C7JLZO9@w4rV_>+pvCrovdX)9w>l>Q$@>w&uwqI(llSwl zfM?Pe%)g`XF=X25UK{|nduJN{#>yD}u7VIrze1G-pV1q!RffVWlRJ3t&%l;DTtP!J zaM6g!!)u!Z6VVQ=Y&Fm51(E zSG1%*w_#Y5+N6?cz7@+*z-aaoLJ+FUV=ag#xrejLOZa$h$u%Wi zO1M!NaW4f#ZPS=M%Oz5k6gAE?sYn--=d<668vKJCE@~x7n2UXi{nSXLDll6tqwNLN zM_CvZrDQXVq)}0U%F!#lT2^8isz3sh*0ME$`cv9al#66ULy{}CzK>Yri!!4w)na?o zlD1=653n4VfoZ(1*I9k4We}EGWl02C^heUI;Qo(#wI%RJ*%p1`|acSHgFXfW&@w}650OcY{CvuGzu%kjW2nQnOw zr@ZkZ!$<1~)LfKsGuc2LROT=bO-YDQl2^8!U9uw0aF@W`LtP?+FH0-&A7GgZ(I?Se zS&kzACfJSymb6>-oOqoJZj2 zM$#)Iv4pQyp3b2nYH*8ZHBXlBC~D`9cY8sj#(xblBUDJp)b7uo<<)%tP|5;DPx7|D zaop<@bidwge_&-cpXI{C*8k$00v*`>6GAKEpe3}CnFnNE zv@n{lk<8Dj`Zb95r>&ESBZ_2QV4-enhr|e9&F~bg#?$W=3w;0S+uXDYb*5wq2fef` zgtJSr$0!MtsW>j5VN5Su>AHLdVV`;Ro@sxGqUze_XH7oR4{O=u=4Wj_+6RGwL>xuE zFgRrxxuKQHYI*J*t#dZFZd+U}42J4Q`;npx(*rQ+vQZpNLvuo_#Ys~b1W0w^OfT%P zwyYRSe`oKpMx_T*NLC~PO8mQEPfHR57uG3S=`5BAsxDyZzUN-2cLgccQ{&VU>kGQ> z`8tk3ri-XNG_;7v0>vsEpMZ_CPMbc*`1r@hF0(8UqpJtQx|L*MW zZtXTVH}L%b_GaVZ{QtdNcR2s!Og@f09;l~)NG@a~wVpIm4+4n>**TzS>=t3rP&pXT zNQEI9;9c&Bg*X@C3?t+Ps|Bn9=#CY)3` z6GAQTFUf%YmB}Xk6C&l$b`;5f_oEjCCOP?$KijeFRK^m;e>j-&?{eMF{wwFxMt3y+ zk8ZaA+l}4A`Oodf_RfR-zmMzT{O4cn{O7kiM)>gEh2>`t1^IO(|ChPA()U*5FTTFy z{6D@WV%_pHKsVa|X0vGjceWnl|9iP~oe2@hiaNql;{ zf(x-~YP7$^k0=AnC+hI2J1H9D6fLw(WEG@IwrcStE%$!<#I%}*e+DNn2g81+`>G-u zWP2Mns?_ZCOk;Q|t#8l#OB*Y1V%cs)MM$ymPg+2WabPxVTq8U^TZ!1o!077-#$l&x z9IVDq!3vO2KJ#+I?J(V7ED(rs-z-bo=N*oUfB@u~w3~{aAkd_L##qHQWe-!wq!j%m z+LA-)safdd{r>B&)6(5 zYqvE13cSB-mUH*-ztLjoT|WPI9_;^pTo3X8Uo8H=1--KHI{Ees z3wRvWxcy-i#lZ+Si`;?~0f!iSg5^8YQ;|V1wBO_|pQ6cQ;t?JWID~(w58_~we=xD# zJ0O~yn?yGBuSQ}YjKFqJEAB1Puvzipx5~0;O&w2#3Ju~kS<16;((_4Ho|G)5yG69TcnBryflL=5cAJ{TISE>U1{?6ZF9sg2pASc35!2-m+S%hH>-Qz#s7n@i4XI9 zZU4W!OY1+4-JPw?ogMQ3yAS^VUanhz{}q_e1oKya{I#NPIwpI82gT*WPMX;%57B}X zVD_@}s?!~oq#om`=j{BI*awhojfSvfh&Q~Zz6F)d+aFZT9$Q;09T@vZZE47IJphZ- zGq%@aMMIW+a#|k!sA$E}w5qUf&lb-2nKIwdw)}9y`o?~*fAFmI(s(5;Fif%&;etM- zb(i?17Sp;=PI?s%pHY{?9?*XOb#6Q`jAV>+Gdt?_j#JmVaG)sJDz6?$QnrA|;qsG6 zJ_Vax$$^>0>vijkc71uP7yk;^ey@FG4EBvm_vGlPb6DXJIk=F$Vlc4I^d@kjK_9iX zdE{X1H5sJ>m4la_=xg` zT@fQFuJP{`BdqZMKnTmVzrz0p=9d5em(2e+cQ=~NhyVY_eOx#5e@?(5^b6dW-+qjn z;=i5dPBH##Y(K<*_i|w%W$eH1vEJb!MOBA}aqzOe|8x2PEm)`f3cMiQzG?S=J~<}a z27d9tIO@FV42?cpXTO~Ezzrqcj@z${LFc~>)}Y&Azw>5r-0K}BaG=i{{FYtP{#T>l zIebUA!bCd}e%=XBwbJJt=mU1WfxN-Qv}-^gY(TqZ@iVhSysN>#{oa9*HR%nM7KiQ5 z(FuftC4tQ%^Q{)WqMF!NS}1w3WmOs;)Ob8jOGh$78&EQfAOYN3Y`7i0Z@MD>4o zUI3GOH#bTik7%y;JWPt>0-IS^PVHg-S?) z6pT%rtG;Y6|LCa|LSWfkMmb&`U1+^GvBK)L~`Av@;OptszPfhNZ|=u%+JY zx%{lcWzV&V6mKz7V+4y_iO4mB6=W3!EsE{@-gtgjfix>>K53zW} z(!X(w8Kl_()X4520}+>K(_s1bPSRPbkJhESCtWayadE0se-9M5s`Yd>0rnoSpLZuJ z{i>?{mNOw_ddQsq0CjL%`ES@H+gcCUwEzG56|Mj5@9yuv+Gpo~_xI!TUr!SLy7Hfv znDEwvZwE3n8-59sk8g}Sv#B$)X*b8y^ak_RaYC$$-19gEEm1H7UeQ3-Xdz0x$&?Nq zn;_P#ZSqSoriv4>na^zO5Mvk8Zg>`uWy7;*u8xrb{oq@Q4XzV(>Y*-F#K+bmWJAKr zPr-zbBR;aKd>{ew_~*1^5yFyo&S9kam6eRotYr{BL3BE!;H zw%Cz&3~bklX1ju9I)^4%VYS%@Uf4liSZg0u8ZCT!-)~jX!+ri87>_l}k7z(PL{dA( zAFX!hsM0KS>K_exl*p`+49V1y4AUkD^L{YIJC&iOf9e$M3qO&*ufX;OjC~=ciz?2X zpfpocDwI&?QO8ji^g*iNAwTp~NtaXE);JxETg{x`R<`vo``RaMLsu~Q2GVKp+&p$h zkqL4OKQ72+$*D23+9Xcs-YU5u!TYV=H5gx)G8OZF%bc6w%~J$*VE#LK<{I~`jpajK zGNV=~e8W6>&;EUxvrquzxk>gi?fEHjKW9RRLdd@vq!uN9d%^oZ67FmNF<_qiEdg(k z|F1AkxBvEE731^YPZDDJ{}}e4gsU=|e(GsR?WkWEKkVV=pYix5hBR$Hrj5r9F-Wwu zAg-B^Z-03^@o}s=EbXw33@Wwe!mH^07U(cmrP+M&#PC=3!%Mtt=L~$yogN_! zJ?-pK7tRCg<{OvI=kY9(%T2~ToNqC&?z?5RY2J1F@BXN%XWUZRS+|$6ZinPAJ^@4X zA{V`bxprO(vt#}24BEoxDb{G!>c2eJtbz*Vta8k=N}58NcM7>ZNA4-)XlqSik^T## znXQ)=HnwF6e-?8)&|VQ9!_r=BKYv@>7!!p)8*TJ*S^?jB5qMksPsMt_?*X6<_TTN zBX@zm-wgS?d}(6u5=jMp2(|I$Yi-`+0$c92c#o& zt44;Q995HyI+C=p3SFPvP|G@9Hu6xEcAZb}%K%YPdWW||gGNK5jiL~Kno_}w6sOqD zI%bqU>#fsg10564Xn7LT=^?XPx2^W;nYG^#HJkL`KyQV1;px((c`KVwp&90XAWZ3&Gd^0`HZZXN+dZxSeZ zZHduyc8#unAOqb1n+~tgE*VZ1HbV#nD$pGP2rtmW-Sg$uaA8w9V+XI8ffZHor+A2a%YB!rmNLcA|G9h4=RqX8_8UI-ax{QUb0bNcc zxC?Vn91##$-5T`!Y;5vxo@lvCAgM6c zw_~O8f-LssL8ew`Lx8$j*~(OEwNAa;#mD1LyVtIQ2rXN^>Tx#nq22AlGa{j5)@%mR zOHfq@0-O~ zkn~a|X*u%-L*QNBrd)N7^N#G#N9LU`(AE1?Li6H9)Gz4QgF%GR36}Z7W}h%*@K(0B zgI?d&i{Zh5GnXRx#N~4wdC1j;gLluBZ-8OXpV`7UJWbw6+)P=%@3EdxI2kN5eI*9B z^H+!*cxE#%15p0@p6XE-F-Qx_Ib7mRS_V64!p(Nh5iv4=v<=E0M_7jH+MPQ<9@Apyj(9)m~y!4`C>!c=een&Oj3u{d8%) zG}I*o#_l7C2@PW1o+_13p15-Cs;CkX$ykH7DJCLOZC`#Zls&DSmMkC9J{op5#eJb&S1P z2x@*tY(z&xM`5KYVyO9$YLxKjl|W;K3iuKoS7Qy-(f$V~4ya0+hBBOMg)T{Zt*%81 z6xU|e!4uTuP`x;Kf#ZjGylTvM5R+8JbvgMWJfP34IsPzF*4P(0_o?ELpeD_)*@LiD0yYSG(N@T065O9g__?V- z$wXaJLR+OzGg$;j*0igekgkQFX~sbX<*nsdKbEc5dwBTahpZtjABs%MvHoKfNYi;y zE<~lo;DN=wg!J%*lnSC3X6enSMcv}1von+#z&N7A1r`E5_Pe0tUf2jf#oXk=7 zh|=%a2kTR=nIMnX0`&`eKua`>XxkQn8u^F~amEj!AFQS=$tKX16ue3c%>e5WoR^xE!oP2|Uero_OD#>| zUj;8u<-&dtkeNEh5tiS_|(@SrC~uX%PnEeG6z>tQaZs> z4@7E%s-W8)Qjz8fK+OlW(8nfMM3D|i&Vje{3oxi}3#XqCKD{`&Exf>RaLYG^L}sl} z-bBJ2#q%G~K*Y;q%dt#}V4~8QS^s1LQM@Vs?fX5WR{l@x3Y3cp^Y6!QOxa_q1Q()3 zXA{RrQe0j!q0)#(EX|RX5mA9$Ta`Z|8#|d3|7i*iM zZXkethF#6LrdVYaye1X7j(%%R7WUlAyT?}g^Xccp!KWV#=&p5%?kY-#nJXT-0m`J% zzkSXt77e8!@-M{_Q7EzMxK1mKH}MISEPwi-?P}${;6kKUl9*gh15+T^W8z}CMqkLOQD{Z zS^-ME2>B#e0Q`jtxm?JW!{v&_dg42dxu}NrDrBlixU79bt zb9*16`GPjEw3NTuR(K%}*}k>LE;POAx^rmx!d*=+nETf5Pt9Y97x@|T1!B^IUhS)9 z9QB)jERruKIW%dKMdcmc7(C56vL zxp#P&G8rXn!eGQYxVZ!(rq(D6P9%Q?>G|#&ws9b!v(>Tl3)PXKB_6h3m09=&5~)H^ zM+y!mM8TEzC)Z*-aU!)(>F-pO z&sg7ihytSquJ#SD-mf}CRc57hr^4V1(aKHhxZtEQI#zn`rtmpc7~iGCJt4VeP+3=r ziL5TLZXMFqeGqbA=N^z_TtpcUtg~(AD(cn>BrVQyM;90doq6ME=FsEcNu4(TMWj== z)VI^ec*c>tnvSW7v>Cmcjpgb*uUk}MT9>D;4daF{E zy_4hm;HYwZ+-Mz&_>YxN^+TmoEP==yT?`kwR_G1nt(TV0r48eHInOoQt-}(!HlQb~ zQ~9}sCwhjtYP0>0MYR0KclgBO-=U(*pCMOf?{&7cT7mJ8KVBC7A3ROJ)Bd?pE4lbm zKD}?YE4>n5g`a~i>EJn(T&-R$jp4`TaEi~3qe|2N-R{%r+z!0&LXU=XH(I?C?toA1 z1wPjg>mB*KQ-`8JtD5GzP-qjAdQd7bA81^oRq31zB52~Vtom?tuW^J{(b2Kex4#@U zaga*CUrt5#AU(&;Zs?sXin)o#pVmLxEoIdBaVPj`{TsMXFh{dptu(jD$0ieWL;Ghj zfuu53qs<()SB&NezxjYW>bRl9?5-i-xd7vG4)&##3h4U33GrJ!G%t}ySq(=n9}VeY5gi0WiTHI3pJaR9qV%u#+#DOXI@eH zx{Qxv9+ja*C9gRiYRpxTF{s31HY2CokpEbm=hjLHaKorFppnqp^AYRUAghe5874^Y zdTt*vmrJZ-GrtTMQMkDQ*}7A~qf?NxFgDjA;%`&z#0sm!3({uIjl|y{{u~GjS z6LlJgz;@tC92>_}op!Ze>vu5S5l`4XX|-D?M_fOlBOiER+d*aKhKM*l_;i;WwsA+6}hrF)r=8=1aRlVm95` zb))Z9>XYZ;w|SrF%I<~>yPL1;Dz`~-u3N_2yXHA2JXsq0InojKmTs`uJHdWHk6>#y z-d~2cfW)a;XcleP6=cKQrz5--sA@hj3XI5uTd4cZ;!y9tqxR^Ae{6Jm{Yn$2ZnJ+B za)%YsaOUjlUTy#4EbyWLcV~ntPU>AfiAt~Wmag7G=&TiVQnudPvFR%Kh%Lc%nFCn4 zj9`zgp1+e9=-K%uw`B9J93h%zr->0@8#32rGh(jFW5QjL#Ar5hf ULmc7|hp!I*1<6WOsQ~Z*0HjUelK=n! literal 9980 zcmVZ-%FzrTxr8~eL;`EPVRX*BnCwzv29>h^~Uc0_RbTw`^7L=bA^7O zdyG8^C!rmLYx>>M{x`kmtNHIbelYh8|Go7mMPW1k@7MR5JB{XU<4JwH-q>k8^8X>O z&Gnd;1Fb#M*hqVG!ba_v#~KTag*y)PiDj_%urSn)wa%!(;6e$wHD_DNP2p(Je?zzK zd<=99=syF+$S=HotqnA4oK4IbYq2ti#T>Z9fj9^SZKC@IquvZ|ZGB-t=V|D904DBG zu2h~E3cpVs_g2O*aF~)KoKyc=VQ;&m*Q_)AmA%&5huQ#!9}fDVKF1Z#VXK(f{x7H}@Mm+i3rrJNwN?{eOr{RawU~KxvsyI^lSr zKP*-*KX5$wab<9YXCMxt1GHDPi_OlCmYUkMo%p;3T#^VpMJNO*N$BBU43dWoArCZ zW{0eN%us489BW4-h9Ghv5OtVq;&9ij(k%=YU=Umz)>+Onq(8_~zQUai$5QGKh?hY| zwvhW%k*8(M9)&8DX<|C>lWOfmD__s`|R8gJky?! z181yzo_OT(SkIrWy5Gm~)S~U%|QhVc>=VL7zHyfDRf*;C+tP z>*O{sP0Pq}xud>Nx0FA4^tgY|>puRUi=+>K!4=*k0=Ucn?=-fv{(paG`_cbD#P#UN z|B{Xz>9uOMj8_9+MpMV#$q9;FtJ6R2jmke%;)F{Nz(Y;KAj-J#e#zy+Lpxls7pxvT zNqA)XGtY5dV^&#L z)zxFF{#DK#>+8SU_5H@4sQ>M6@6{nr*r_+`dyoGAA+D$2)h4E0o9O;k;VC*EP)*OA zUj<+qD(ozr>zE}TzT9G6ds;0#rB27@)UbWs2KGqmapHdEHe3%+;K#7gBa4l1}!8}p|6eKfL3CNiM z;!FY^`s<7=i%4ulPq5yI_J5)Ax(WjKpjNxNxvA=eaMkhVHH&xjYscM=)*EUnAT3sU zYFi-VEY7wjw@h~drzzPjOTS@`$8^vVj6od4`No5o!Jcn1-?g;(4I?v|Lwc**Aow|m% zFnwgK4=BzEj%6;)fHF;gd1=)`!8Zb?LLxcr47w+y@nLt+DwWxE#vm;%!mpX>kt?Cw zKVY@5$HV^VprakArTV8Ls}$MyFIa76T-WT-ve@_EGxuf&em{cPYroL|Bvy$DP^n%i z6kcn`C*zU!+o)A6ocgee16Gpyvmen(JGKpB7gb2ecfDXgN-hiT5)(I$OxzgoeNiZM z9Ls{SK0D!7a`?+Ji;LM<1qT4;l&BPC2n5R6a+U zFflO?bUPp@3$Rpdf_$ z0D1&^OT7E(3ZCMR14Otso8WI{JtU^te&`9kk1+9>3sIKgS-0$c=mcD;^lJ>w{3`>P zZ`u%DgFJJAg8_bM_&^E9B*raGbz-`P>KhQy>j;zj0KxeXpCte0%A8)M80S%3GQolg zu^K6U$BI7apkTmO#SJH5BR={%SdWjUeGNQvyjunFTp>~f0L1}*!(`B#x!`62FGCby zlYuN62v!kjGU3&uyYU+lsi&R^)|Ey=Tnq72fa{#{B5;txz`SH;z-Er&lbJR?0Bwla zrsMkKsqO~A!7=gVTDPJZA{j<@hCb&ZNZqA555-dA5lH!kp@SeqwXbgfWXSGVN3wnt zVkh*#Ka@d=eFLVM`IRaw9~zf>Xa%gXU9YT#RwzVVgKj6UPTIra+y3CN^`i^)z5(5Q zk+vJl3h#vy*h}^=t@+$geEixqiWkzG8;t~OE8m|i*1BQ~x@{`3P#Nxg7sCKCWC z9Rw`)n~7aRy#gh_HfP2xn(=9oslcqAoVg;~UH_97o1N#%?1i*~Mgg*_1fhArn%^2B zPSPQG(3ucu92DzMAs0jogdRrq!D)Ly<#?!X8HSta9F{ZIpt}wr3 z+x#u;B1|39pGfc>Q)fVj6(jQ$#-6=Cc=qPt*%0!6c3vXL7px>e6xl~e-(iRj#EsTaWKyVC_yo)c zZU66vN68qDjIVXi#7w{!etNo?aSI1J zXJXw@k764h30hbjz8ExySFZ5BqQAfh2AIn?NLtp{~DXC(lD9^!Z9hu;-;alt-* zOg%3C@H;4ltQ8XRp_5>S1>*jcIhdo|n3yu^vspAdpA#tQgedzl(ivz9f>Ss=&;;Tc zb_T--_(PN!dNxnm_{JBfB}5AUB*K@XC)+Q+Z{$kjKiLg%aBc+LWX)|1vuF}bRwL;k zwt1l)u`t6m@idX>r;Nm(i;goFgaF^~)!>g4 z2GI-=Y$DBOON$xGN`IuV{=`(^lgH@$Vg~@wPqa0sG(TR5UqAtpd$1xqTb`#w<#{(+N7IO1w}| z%$U4xph)GYY*2*^@@rUv0_zERv) z{5LC~)q0gB35eRq0Am_b>JYUrHv!Lt>0fT65Wr)~`9bIcwg(pq{>JJC{;q%!!59c* zX>EWQweaH(Q!DKb1RGRqDZ`qPs&X2AWY&Mkyk!qg-P-XV;d>Iy$G&GHtMUx|Jqd+9Z-Wr3y23 z<;A^3`5;)7@lQjN!ekVXmHigiEK4ffH(q+E3M8;G1z;uqLo8q*`l$Qs3l-!wc^h9O znwd17PkH@Cc(TZ8i+U&)bx~03<>jS8(PdyRl6tAXl^#p(ehi*-Ec`qYi+g|R zX$lQqm07Wx)pYrWat4X?W>Q8SzpacB3MmQ0H1``&aua%S$bXtZ!V1nD+eig9q&A{v z!LIpsa1fTIw*gFC;S1flr_ZRcA9aOEfK*rJ{L1jE ztMX?0yLgAGsOkz?SQGY0t|w>vF>*hdl2R*P#R_>;IV{z&t=sr6BaCVsSGIU5NL6h| z#Tm%!(w%c-L*Pz70sU*!GUkS8;u#7aiNXh#qtDc9Q&;1|c&g_(I13&XXo(X>HU6gx z@~IS;1R{}mu{uc*Qqr7}_H389T!CdJshqU)(?HjW}d`CxwAg5T zM5OfD&Mf)wfAo^TL?=JfXFFG&%3PuNj|VgVU9QdSzjQu*d{5*5Un=A-?8kV{n=7lEWwz|FQ4c5g-RIq@6XeI`@2!g+|^Sa8U=kZDzp_E-23X=3q2 zo<4RbSz{cdg|>;Tf-uR`YB&p`DL;N>N>#x>!_$|;(V*LVRpuSYrqJHlc7?i~UnmSu zr4NjSb8TQbhZktaRM{ds!$=FL>#ahHphuT0r)Q-A6?QlJQ5>|kO^qTjn zZ+P(mBaT3b=Vn;cKkaa21UMjDr=56uftQzclcu1u{Y$C7>g8fV*t*Wx-5dW6v>b8ybHJ-rqILx&QZH zsQP`M&%fOH{`c-qw*J%HZajYf=|Qf?`2Q~!|8GFAB)pE^{bsHo236m9KMq27jN5`6 zkizvL#-3sMPPb)b5De`s-se^HG$uCTVS+>Whx{OPXXyts!@38eskI4YJ?Cc3cRV?^ z2TFDWQNeE6hhHnpqB92gRtVQ1pB77b7A|@@$x4%w#k5=jS+}gQIJ<>Z;ec0Wa2aC$ zxmU~oOx_hIZ(lb@{EUDhFG|S&%uTfE`)^kFypR6}UE?q2e`)`}&+C8neaHa!_Q?P5 zKl=X%xi%U*~{*$Zf{f&dW>&9(Sdhq zb2&O;2w}+xZ~Us<3M!knKd730Rw@+^wa#%{7_!s|z~cOZ9kf{1kVW606vu|kN_IA- z!mZo$-1R<}*6SHl$D6T%*69xppBG+guY?7LMRo(+C5E)_8sF5`r&q~ty|j3GW0i&+ z^?~-C*QxoyFp@D&t?anpKZ#vy?nInjVxH(QyL7HVOqX6n^eMRLiVj=n@7HZD9jRlP zFmieGFLQPJ?PG1&(aOEklCdOW~aV$_Nc=0*# ztcUhf-RNs~%lNi>8d77$$#Th2N^FXIrZ^LE#|DIgWDSY7k$1uaV2-3@so~mh zTIUqhk>}*)B_b-_-cTEifOViBvZAi6_QV9NuwUE9r`nK}pB-;8`sY`aWj@IwQeDr| zoUZwjlInODbA=WXIUq1V5V}HBx`2Ft(x$frwl7K_#=USn^`5bq}e|z-(pH%!;-+he#9^}G6 zO6$Dtv;NT$MO8GzK#I56f7e#<86|Eo6W9-YxGH_>K*pLf7FqUd!_ z`hZP;C~hz@?P)LuJJ1nK{LI`4@2c>x(?8UbF8!g@nKJuc(cGo?cFQC(_yHiUE-i&F#b z2r%w-G^J92d=Jde%QG2&In;hGj1U?@z2JPnfskkA3>$nhAMecc=f=U!T zftMnXHBpGvo|j559h)H2oZ94%-I(l7qRo7sV=o!@$lK|Yj4#tCtF1aVX5xdd={7`6 zFwmArD7lY~UC13B+rMNxei`>sk^KWXh%bN6>~8zN;t2RXvcS9Yzvln9cctHHBT4*w z{t6947D&Lv#7W*`*85C=*u3Q$j%2bpvX&;&th1KQ| zWML0wVXf1uG}`#|YS^yg4Y!4t;5^nG^`QaD5J{zrKiZvMtI{m?>aQ(nw2wI}8B(Yt z8MZ}^GjOh!ixWJk7n=H~M!~jFk<4`kwl-jG3#nanb>;@OnI2Q4gc^?;j+$TyS_L=x z;Z0R^;*l+l)5f^f%>8vWasD&;tVoM#3Z~ycJB=RN`_8H|L2u#Uf>e%VjhWLSbwYA*Q2n)E+|^K~g>vD@$1F?Kx8h?hFE`@4AJ8oyZ+^M|rzLL*Zc!#w(6^7}IHpaRBy zlk8>2S21xtXG)tw=x!W{Epq(&iueB{+}8hN7xUk43OGjpKf`aP|F`$7l$`&(myqcH zZTNo@uEOg2>AN9~t$tzrkh{Bo*6o)VGPL-Zv2HiSAkk8SxF$ls{pJ0{?Ra&V+hMN* zV~9@-YNh7Gv*^P%*f36|*}OAjs9F8+6z^J|fw9~h5W>Lya*sZ79+)?eoi?8*TqLKP ztZO)5gRgG8WEIDE{o%`R#d_8ym7R5KDeG29{^AT6;){Ir3g%kA6mG|Q?hN|Eb{1>2 zYxQ5Y#Z^$C}{h8?4x>Nb+gUf!p^qL;d^nn1N9YQ8=m%B z|M~0s#uzCyY>eK^8yS4#wtC0}T_fXn7Rd?jgHbx2g5(+pXUa zHF5TDptVWEGcL`C87^;S5L5sA_hC@S7{B*m_V$OZEN8)7hzD_LVab3_PG*QZMu_us zI!3~bTLvM51P~RcD-z=BI$Mng$1B=jPxPq1aD0!hIAr(PLt&RpDvRqAZI-vT_!QzW zH9(lrEjd2g*tQ6;Kkb^653W zWdFy#gl~rb`}Uskf4S`V&JlpWchKIuVfHflvkr6-GusTemv0gHwwS$96+%ws1g^pB z#XLDa0g&itXC8x7<1dgLe>F#X7!sX_*oMF5VjzB<-5n<*k$tnZI_m_J z`+a=u_Bw-36;$ZHGpKfR*`rQ>0M7yoIG08E5H!_+fLtzH?X=rqa*S$~K?Re8Q~nx8 z8|zxIx+aTHz<$(&xm+#|J-BpFL^f(jn>*fsq^CMb%b7nq0omn$8yI97RKb9pOCLDn zavt9t{`kS40=`%S{O$BA)n!Di1T{M&F@iEgy+vhZ z7-DGokZJTl@fF7=!2YXR3$F6#G z@Cf@4u^ja;KfsriuCDh}kHQ`Lqf=*SK}3lxQtf9DR-8hyyb?wH_~_t<16ZO^&7{;+ zmgs|xNxtRsO%1FbmG$YdH^u@T|@Y9O0LT}t95Wsb8s z1lBaHtE-T$g~K%KqJi?0<2rxeciOMu;ql{~B`)uZY}$4H>lDe-d9+`QiiyDok9i64 zVT9zJ)P734uq7Cf3HMKqOYftjO#Rhqw)o-LdAEH3_?rDo7a`+Ro{~r8e!~W=qTFyn zwpRmv1l^$~hDWq%jlhiDVny8XUFZj|X+yHfHr^iHD6#H9IQfDRl!xA$Z_Hn~92l+; zR@>;?h)R>0aPFjNKs@wKwu0YO%ZEPad6QpQCBnqAhTpI}=zUu;?l>xUvnethmlXrZ zd|D1zaT%s*DiMXLAj<~^Wj#a1tJTnshZ!MxsY+@5`;*a%MvgyGP1E>S17Qo|_ia!D zv|AefSn~@20P7m{kZa1ZRtESK>W5+Q>90dZ!-B3Xml)pcf~zPk{eZ74Vk1*tMdTdX{{a<5 z5H>a)%Zv!7D&3j$Z>A8%o6tr_htjk zMSQ^k^>QgqJg};%4knI8j7?e+V4&o_hlUaBWkI<~5J<0gGksdwXy4qsncYE?nVg+0 z;k}V4CWQ&o$YJQeQ2mKdft|whYB^z3W6)Uo@9rlp4^5ZhykDe*u;FBx=p(L-EG?a=+9f~EQ zP-E3`o>n+-VhEHS#Xjh}It4#C5ozQkn2U8LgEt_N1zm*eJ_P0v$be)UF5pKv1m zD@ZT&*NBY+0lTdCoyydN8I$K7l@=QK@)y97!ehx-l!?vxbA}8S(q=pGiYQY zWeGx0*@4PXp{=R3E|T#bTs-+KmJ_Ga`jq#biu{@AI}cG{wZQeZ;l-<>vs7eGYIn*E zMu?Gb#>Ai|i_vv5yH~{z>Eie%6K)A9ErZ6oPE2HRfqCmS$kxi{9?`|P2wfeRXN#9A z`qBzSEzNO77X$|FdE*e-(BeNxo3`*t#M775H?zlh!ZUAmF{UcgV)SY@mb3G`Y*B=5 zUY?pZ3`V`U1WW#0tRjNlu}ySpy}ge*tvYY;l+-!+vRud%SN@`S;hu`!N~PW6*@X*o z=J{TR>l>Z`k(Wfb2^~h?`_-d*t0F!HuWYV0`nWZ8RPD4{_4Yu}!L4}lRqkID-=DDW zQ0NHCe^spq5<-GXvsrJBXya*LeBI;ORtA;P;BB`)YE`=3M*C2Fe_iQSk1D-V8C2fc z$CE|A9eM+4>*b|;Ie~t?oadXJ_F);XHlQb`S9w#$9X%)cYP0i_eQ2x4m-xiKzeGcq zKSQeQ?(^RzA3V;m*Ll;Zl|6hZpI$XPl|h-$!q35$ba5X_zE-c6$MECw zpl6qSBI>?s=CPcA=4&C^&wXf zK2W(vyV83*ilB*qWyOa-4;n4>idtQ*ZGUMsv6ITbUtWmhL3+B)e(0Udid_?pKb?Pd z+FGyi?N0F1`2(a+Fh#Rdtu!}C$CwG4q4P#CkTj-hw3x%ziqRe6Cm(P{9T#+%-*v)Q zF2K2*gMTTt0$%;!1U@5Y2$+9xw4W+YmS@_k9BwI_6qr=m0a#b>Iq#&P45kBRp(m5NV|^;Zc+-+8;T4sx&*Uhk(FIym^qT9V#asm)gGMYSvr4*{ z^j)!^+sGjx4WsgaN((H+c?JRb*lB+u!r$lJYxTC zyVHK#;`Rw``M?d^E*djeM8xsI`R)vV(ZJaX?h{t2Ni1abpp3_d_at;Va% z`RK$Sfj5u)P7C1N!+#huG5lc=qI|}UTZAQ4t)XhfGO{*^9!csLoQ)PL)LXby7q6Gn zcDtP8JP>Z_<9lvtJB2vYGWX_VV~@y}Pdw0$=?Mssl$eW*+5a8C|KA3Vyxsesb{YPI ze*Smw#cs0y|2{%;{`bqy|3*ND-|C$1nuEFC7ah!nR4Vu2xm@|terDF5Hgqc2$VEF6 z>*2Z2FDbxsCO4{#bw_g7mQOePhjMMdZkDpaborz|!>g zcBgi|b*FaCO4<(X2J5wrL%W{+&~A{JxHG$6^tnotdG3Ci_nwaI#vIs^rmw)+YA+GSme$PR&8LXtSmuALcd<;jh3{Q^aU6 zB6qH#d2bendRG^CX* z>9wBHH5&*oYXzH>&G+^ubQJuIO~Lfp1+a1&VKR0GY9%kwvh%oW$>tkfglN~DHbH=u zSL2-nJj)lF$k{wQE=YJ@?3XR0Ffg>ehfQH@>F4>Ho4~9@lDt#3&P$r_=R*18Aqh!HLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0eKhyMY2TAxb* G@Bjd~gPG+3 diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db-queries-summary.psql b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db-queries-summary.psql index f63695ea91..02cab1e132 100644 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db-queries-summary.psql +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db-queries-summary.psql @@ -3,7 +3,7 @@ SELECT queryid, query FROM - pg_stat_statements + :schema_name.pg_stat_statements WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()); diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh index 00a6cff831..bd82896db2 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh @@ -118,6 +118,29 @@ run_command() { fi } + +# Function to convert schema list to an array and ensure 'public' is included +prepare_schema_array() { + local schema_list=$1 + local -a schema_array + + # Convert the schema list (pipe-separated) to an array + IFS='|' read -r -a schema_array <<< "$schema_list" + local public_found=false + for schema in "${schema_array[@]}"; do + if [[ "$schema" == "public" ]]; then + public_found=true + break + fi + done + + if [[ $public_found == false ]]; then + schema_array+=("public") + fi + + echo "${schema_array[*]}" +} + main() { # Resolve the absolute path of assessment_metadata_dir assessment_metadata_dir=$(cd "$assessment_metadata_dir" && pwd) @@ -147,45 +170,65 @@ main() { fi # checking before quoting connection_string - pg_stat_available=$(psql -A -t -q $pg_connection_string -c "SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'") + pgss_ext_schema=$(psql -A -t -q $pg_connection_string -c "SELECT nspname FROM pg_extension e, pg_namespace n WHERE e.extnamespace = n.oid AND e.extname = 'pg_stat_statements'") + log "INFO" "pg_stat_statements extension is available in schema: $pgss_ext_schema" + + schema_array=$(prepare_schema_array $schema_list) + log "INFO" "schema_array for checking pgss_ext_schema: $schema_array" # quote the required shell variables pg_connection_string=$(quote_string "$pg_connection_string") schema_list=$(quote_string "$schema_list") - print_and_log "INFO" "Assessment metadata collection started for '$schema_list' schemas" + print_and_log "INFO" "Assessment metadata collection started for $schema_list schema(s)" for script in $SCRIPT_DIR/*.psql; do script_name=$(basename "$script" .psql) script_action=$(basename "$script" .psql | sed 's/-/ /g') - if [[ "$script_name" == "db-queries-summary" ]]; then - if [[ "$REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" == "false" ]]; then - continue - fi - if [[ "$pg_stat_available" != "1" ]]; then - print_and_log "INFO" "Skipping $script_action: pg_stat_statements is unavailable." - continue - fi - fi + print_and_log "INFO" "Collecting $script_action..." - if [ $script_name == "table-index-iops" ]; then - psql_command="psql -q $pg_connection_string -f $script -v schema_list=$schema_list -v ON_ERROR_STOP=on -v measurement_type=initial" - log "INFO" "Executing initial IOPS collection: $psql_command" - run_command "$psql_command" - mv table-index-iops.csv table-index-iops-initial.csv - - log "INFO" "Sleeping for $iops_capture_interval seconds to capture IOPS data" - # sleeping to calculate the iops reading two different time intervals, to calculate reads_per_second and writes_per_second - sleep $iops_capture_interval - - psql_command="psql -q $pg_connection_string -f $script -v schema_list=$schema_list -v ON_ERROR_STOP=on -v measurement_type=final" - log "INFO" "Executing final IOPS collection: $psql_command" - run_command "$psql_command" - mv table-index-iops.csv table-index-iops-final.csv - else - psql_command="psql -q $pg_connection_string -f $script -v schema_list=$schema_list -v ON_ERROR_STOP=on" - log "INFO" "Executing script: $psql_command" - run_command "$psql_command" - fi + + case $script_name in + "table-index-iops") + psql_command="psql -q $pg_connection_string -f $script -v schema_list=$schema_list -v ON_ERROR_STOP=on -v measurement_type=initial" + log "INFO" "Executing initial IOPS collection: $psql_command" + run_command "$psql_command" + mv table-index-iops.csv table-index-iops-initial.csv + + log "INFO" "Sleeping for $iops_capture_interval seconds to capture IOPS data" + # sleeping to calculate the iops reading two different time intervals, to calculate reads_per_second and writes_per_second + sleep $iops_capture_interval + + psql_command="psql -q $pg_connection_string -f $script -v schema_list=$schema_list -v ON_ERROR_STOP=on -v measurement_type=final" + log "INFO" "Executing final IOPS collection: $psql_command" + run_command "$psql_command" + mv table-index-iops.csv table-index-iops-final.csv + ;; + "db-queries-summary") + if [[ "$REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" == "false" ]]; then + print_and_log "INFO" "Skipping $script_action: Reporting of unsupported query constructs is disabled." + continue + fi + + if [[ -z "$pgss_ext_schema" ]]; then + print_and_log "WARN" "Skipping $script_action: pg_stat_statements extension schema is not found or not accessible." + continue + fi + + if [[ ! " ${schema_array[*]} " =~ " $pgss_ext_schema " ]]; then + print_and_log "WARN" "Skipping $script_action: pg_stat_statements extension schema '$pgss_ext_schema' is not in the expected schema list (${schema_array[*]})." + continue + fi + + psql_command="psql -q $pg_connection_string -f $script -v schema_name=$pgss_ext_schema -v ON_ERROR_STOP=on" + log "INFO" "Executing script: $psql_command" + run_command "$psql_command" + ;; + *) + psql_command="psql -q $pg_connection_string -f $script -v schema_list=$schema_list -v ON_ERROR_STOP=on" + log "INFO" "Executing script: $psql_command" + run_command "$psql_command" + ;; + esac done # check for pg_dump version From a51fb59c747e7185bb1cdf8c83ad1d92daf0f483 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 20 Nov 2024 16:21:02 +0530 Subject: [PATCH 004/105] Handle MVIEW / VIEW's Select stmt, sending the plpgsql issues information to callhome and yugabyteD (#1911) 1. Handling the MVIEW/VIEW in the GetIssues of queryissue package to report the issues in select stmt of them. https://yugabyte.atlassian.net/browse/DB-14089 2. Added analyze-schema tests for MVIEW/VIEW. 3. Sending the information Unsupported PLpg/SQL objects to callhome and yugabyted payloads https://yugabyte.atlassian.net/browse/DB-14043 4. Enable the Unsupported Plpgsql objects feature in assess-migration by default 5. Fixed all the automation tests of the complex schema for this. 6. Added a few integrated tests in pg/assessment-report-test --- .../dummy-export-dir/schema/mviews/mview.sql | 13 ++- .../dummy-export-dir/schema/views/view.sql | 15 ++- .../tests/analyze-schema/expected_issues.json | 60 +++++++++++ migtests/tests/analyze-schema/summary.json | 16 +-- .../expectedAssessmentReport.json | 3 +- .../expectedChild1AssessmentReport.json | 3 +- .../expectedChild2AssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 52 ++++++++- .../expected_schema_analysis_report.json | 90 +++++++++++++++- .../expectedAssessmentReport.json | 100 ++++++++++++++++-- .../tests/pg/assessment-report-test/init-db | 4 +- .../pg_assessment_report.sql | 62 +++++++++++ .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- .../expectedAssessmentReport.json | 3 +- yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/cmd/assessMigrationCommand.go | 35 ++++-- yb-voyager/cmd/common.go | 4 +- yb-voyager/cmd/constants.go | 2 + yb-voyager/src/callhome/diagnostics.go | 6 +- yb-voyager/src/queryissue/queryissue.go | 39 +++++-- yb-voyager/src/queryparser/query_parser.go | 63 ++++++++++- 27 files changed, 538 insertions(+), 57 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/mviews/mview.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/mviews/mview.sql index 45097666b7..50b78580c1 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/mviews/mview.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/mviews/mview.sql @@ -2,4 +2,15 @@ CREATE MATERIALIZED VIEW test AS ( select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg FROM test1 where t = '1DAY' group by x - ); \ No newline at end of file + ); + +CREATE MATERIALIZED VIEW public.sample_data_view AS + SELECT sample_data.id, + sample_data.name, + sample_data.description, + XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data, + pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired, + sample_data.ctid AS row_ctid, + sample_data.xmin AS xmin_value + FROM public.sample_data + WITH NO DATA; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index 8dba0d3d11..d018df373f 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -19,4 +19,17 @@ CREATE OR REPLACE view test AS ( --Unsupported PG Syntax --For this case we will have two issues reported one by regex and other by Unsupported PG syntax with error msg -ALTER VIEW view_name TO select * from test; \ No newline at end of file +ALTER VIEW view_name TO select * from test; + +CREATE VIEW public.orders_view AS + SELECT orders.order_id, + orders.customer_name, + orders.product_name, + orders.quantity, + orders.price, + XMLELEMENT(NAME "OrderDetails", XMLELEMENT(NAME "Customer", orders.customer_name), XMLELEMENT(NAME "Product", orders.product_name), XMLELEMENT(NAME "Quantity", orders.quantity), XMLELEMENT(NAME "TotalPrice", (orders.price * (orders.quantity)::numeric))) AS order_xml, + XMLCONCAT(XMLELEMENT(NAME "Customer", orders.customer_name), XMLELEMENT(NAME "Product", orders.product_name)) AS summary_xml, + pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired, + orders.ctid AS row_ctid, + orders.xmin AS transaction_id + FROM public.orders; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 1ee5b52e4a..b88b235326 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1373,5 +1373,65 @@ "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "MVIEW", + "ObjectName": "public.sample_data_view", + "Reason": "XML Functions", + "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "MVIEW", + "ObjectName": "public.sample_data_view", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "MVIEW", + "ObjectName": "public.sample_data_view", + "Reason": "System Columns", + "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "public.orders_view", + "Reason": "XML Functions", + "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "public.orders_view", + "Reason": "Advisory Locks", + "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "public.orders_view", + "Reason": "System Columns", + "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 4e710c672f..7486e054da 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -37,20 +37,20 @@ { "ObjectType": "FUNCTION", "TotalCount": 1, - "InvalidCount": 0, + "InvalidCount": 1, "ObjectNames": "create_and_populate_tables" }, { "ObjectType": "PROCEDURE", "TotalCount": 6, - "InvalidCount": 3, + "InvalidCount": 4, "ObjectNames": "foo, foo1, sp_createnachabatch, test, test1, add_employee" }, { "ObjectType": "VIEW", - "TotalCount": 3, - "InvalidCount": 3, - "ObjectNames": "v1, v2, test" + "TotalCount": 4, + "InvalidCount": 4, + "ObjectNames": "v1, v2, test, public.orders_view" }, { "ObjectType": "TRIGGER", @@ -60,9 +60,9 @@ }, { "ObjectType": "MVIEW", - "TotalCount": 1, - "InvalidCount": 1, - "ObjectNames": "test" + "TotalCount": 2, + "InvalidCount": 2, + "ObjectNames": "test, public.sample_data_view" }, { "ObjectType": "CONVERSION", diff --git a/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json index 430ba37144..a238ee8513 100644 --- a/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json @@ -1356,5 +1356,6 @@ "There are some BITMAP indexes present in the schema that will get converted to GIN indexes, but GIN indexes are partially supported in YugabyteDB as mentioned in \u003ca class=\"highlight-link\" href=\"https://github.com/yugabyte/yugabyte-db/issues/7850\"\u003ehttps://github.com/yugabyte/yugabyte-db/issues/7850\u003c/a\u003e so take a look and modify them if not supported." ], "MigrationCaveats": null, - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json index e9da30ab25..91100541e7 100644 --- a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json +++ b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json @@ -543,5 +543,6 @@ "There are some BITMAP indexes present in the schema that will get converted to GIN indexes, but GIN indexes are partially supported in YugabyteDB as mentioned in \u003ca class=\"highlight-link\" href=\"https://github.com/yugabyte/yugabyte-db/issues/7850\"\u003ehttps://github.com/yugabyte/yugabyte-db/issues/7850\u003c/a\u003e so take a look and modify them if not supported." ], "MigrationCaveats": null, - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json index 8d5912bf38..d67d01860e 100644 --- a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json +++ b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json @@ -877,5 +877,6 @@ "For sharding/colocation recommendations, each partition is treated individually. During the export schema phase, all the partitions of a partitioned table are currently created as colocated by default. \nTo manually modify the schema, please refer: \u003ca class=\"highlight-link\" href=\"https://github.com/yugabyte/yb-voyager/issues/1581\"\u003ehttps://github.com/yugabyte/yb-voyager/issues/1581\u003c/a\u003e." ], "MigrationCaveats": null, - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 3525de0baf..fe91da122f 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -58,7 +58,7 @@ { "ObjectType": "VIEW", "TotalCount": 87, - "InvalidCount": 0, + "InvalidCount": 8, "ObjectNames": "hr.d, hr.e, hr.edh, hr.eph, hr.jc, hr.s, humanresources.vemployee, humanresources.vemployeedepartment, humanresources.vemployeedepartmenthistory, humanresources.vjobcandidate, humanresources.vjobcandidateeducation, humanresources.vjobcandidateemployment, pe.a, pe.at, pe.be, pe.bea, pe.bec, pe.cr, pe.ct, pe.e, pe.p, pe.pa, pe.pnt, pe.pp, pe.sp, person.vadditionalcontactinfo, pr.bom, pr.c, pr.d, pr.i, pr.l, pr.p, pr.pc, pr.pch, pr.pd, pr.pdoc, pr.pi, pr.plph, pr.pm, pr.pmi, pr.pmpdc, pr.pp, pr.ppp, pr.pr, pr.psc, pr.sr, pr.th, pr.tha, pr.um, pr.w, pr.wr, production.vproductmodelcatalogdescription, production.vproductmodelinstructions, pu.pod, pu.poh, pu.pv, pu.sm, pu.v, purchasing.vvendorwithaddresses, purchasing.vvendorwithcontacts, sa.c, sa.cc, sa.cr, sa.crc, sa.cu, sa.pcc, sa.s, sa.sci, sa.so, sa.sod, sa.soh, sa.sohsr, sa.sop, sa.sp, sa.spqh, sa.sr, sa.st, sa.sth, sa.tr, sales.vindividualcustomer, sales.vpersondemographics, sales.vsalesperson, sales.vsalespersonsalesbyfiscalyears, sales.vsalespersonsalesbyfiscalyearsdata, sales.vstorewithaddresses, sales.vstorewithcontacts, sales.vstorewithdemographics" }, { @@ -1707,5 +1707,53 @@ "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "XML Functions", + "Objects": [ + { + "ObjectType": "VIEW", + "ObjectName": "humanresources.vjobcandidate", + "SqlStatement": "SELECT jobcandidate.jobcandidateid, jobcandidate.businessentityid, (xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Prefix\", (xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.First\", (xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Middle\", (xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Last\", (xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Suffix\", (xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"Skills\", (xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Addr.Type\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.CountryRegion\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.State\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.City\", (xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(20) AS \"Addr.PostalCode\", (xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"EMail\", (xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"WebSite\", jobcandidate.modifieddate FROM humanresources.jobcandidate" + }, + { + "ObjectType": "VIEW", + "ObjectName": "humanresources.vjobcandidateeducation", + "SqlStatement": "SELECT jc.jobcandidateid, (xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Level\", (xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.StartDate\", (xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.EndDate\", (xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Degree\", (xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Major\", (xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Minor\", (xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPA\", (xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPAScale\", (xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.School\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.CountryRegion\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.State\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.City\" FROM (SELECT unnesting.jobcandidateid, CAST(('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || unnesting.education::varchar::text) || '\u003c/root\u003e'::text AS xml) AS doc FROM (SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education FROM humanresources.jobcandidate) unnesting) jc" + }, + { + "ObjectType": "VIEW", + "ObjectName": "humanresources.vjobcandidateemployment", + "SqlStatement": "SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.StartDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.EndDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.OrgName\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.JobTitle\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Responsibility\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.FunctionCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.IndustryCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.CountryRegion\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.State\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.City\" FROM humanresources.jobcandidate" + }, + { + "ObjectType": "VIEW", + "ObjectName": "person.vadditionalcontactinfo", + "SqlStatement": "SELECT p.businessentityid, p.firstname, p.middlename, p.lastname, (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber, btrim((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS telephonespecialinstructions, (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street, (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city, (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince, (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode, (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion, (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions, (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress, btrim((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS emailspecialinstructions, (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber, p.rowguid, p.modifieddate FROM person.person p LEFT JOIN (SELECT person.businessentityid, unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node FROM person.person WHERE person.additionalcontactinfo IS NOT NULL) additional ON p.businessentityid = additional.businessentityid" + }, + { + "ObjectType": "VIEW", + "ObjectName": "production.vproductmodelcatalogdescription", + "SqlStatement": "SELECT productmodel.productmodelid, productmodel.name, (xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1]::varchar AS \"Summary\", (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar AS manufacturer, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(30) AS copyright, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS producturl, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantyperiod, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantydescription, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS noofyears, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS maintenancedescription, (xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS wheel, (xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS saddle, (xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS pedal, (xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar AS bikeframe, (xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS crankset, (xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS pictureangle, (xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS picturesize, (xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productphotoid, (xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS material, (xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS color, (xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productline, (xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS style, (xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(1024) AS riderexperience, productmodel.rowguid, productmodel.modifieddate FROM production.productmodel WHERE productmodel.catalogdescription IS NOT NULL" + }, + { + "ObjectType": "VIEW", + "ObjectName": "production.vproductmodelinstructions", + "SqlStatement": "SELECT pm.productmodelid, pm.name, (xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1]::varchar AS instructions, (xpath('@LocationID'::text, pm.mfginstructions))[1]::varchar::int AS \"LocationID\", (xpath('@SetupHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"SetupHours\", (xpath('@MachineHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"MachineHours\", (xpath('@LaborHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"LaborHours\", (xpath('@LotSize'::text, pm.mfginstructions))[1]::varchar::int AS \"LotSize\", (xpath('/step/text()'::text, pm.step))[1]::varchar(1024) AS \"Step\", pm.rowguid, pm.modifieddate FROM (SELECT locations.productmodelid, locations.name, locations.rowguid, locations.modifieddate, locations.instructions, locations.mfginstructions, unnest(xpath('step'::text, locations.mfginstructions)) AS step FROM (SELECT productmodel.productmodelid, productmodel.name, productmodel.rowguid, productmodel.modifieddate, productmodel.instructions, unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions FROM production.productmodel) locations) pm" + }, + { + "ObjectType": "VIEW", + "ObjectName": "sales.vpersondemographics", + "SqlStatement": "SELECT person.businessentityid, (xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::money AS totalpurchaseytd, (xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS datefirstpurchase, (xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS birthdate, (xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS maritalstatus, (xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS yearlyincome, (xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS gender, (xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS totalchildren, (xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numberchildrenathome, (xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS education, (xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS occupation, (xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::boolean AS homeownerflag, (xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numbercarsowned FROM person.person WHERE person.demographics IS NOT NULL" + }, + { + "ObjectType": "VIEW", + "ObjectName": "sales.vstorewithdemographics", + "SqlStatement": "SELECT store.businessentityid, store.name, unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualSales\", unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualRevenue\", unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"BankName\", unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(5) AS \"BusinessType\", unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"YearOpened\", unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"Specialty\", unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"SquareFeet\", unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Brands\", unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Internet\", unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"NumberEmployees\" FROM sales.store" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + } + ] } diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index 78ea5ad73b..10b779cece 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -59,7 +59,7 @@ { "ObjectType": "VIEW", "TotalCount": 87, - "InvalidCount": 0, + "InvalidCount": 8, "ObjectNames": "hr.d, hr.e, hr.edh, hr.eph, hr.jc, hr.s, humanresources.vemployee, humanresources.vemployeedepartment, humanresources.vemployeedepartmenthistory, humanresources.vjobcandidate, humanresources.vjobcandidateeducation, humanresources.vjobcandidateemployment, pe.a, pe.at, pe.be, pe.bea, pe.bec, pe.cr, pe.ct, pe.e, pe.p, pe.pa, pe.pnt, pe.pp, pe.sp, person.vadditionalcontactinfo, pr.bom, pr.c, pr.d, pr.i, pr.l, pr.p, pr.pc, pr.pch, pr.pd, pr.pdoc, pr.pi, pr.plph, pr.pm, pr.pmi, pr.pmpdc, pr.pp, pr.ppp, pr.pr, pr.psc, pr.sr, pr.th, pr.tha, pr.um, pr.w, pr.wr, production.vproductmodelcatalogdescription, production.vproductmodelinstructions, pu.pod, pu.poh, pu.pv, pu.sm, pu.v, purchasing.vvendorwithaddresses, purchasing.vvendorwithcontacts, sa.c, sa.cc, sa.cr, sa.crc, sa.cu, sa.pcc, sa.s, sa.sci, sa.so, sa.sod, sa.soh, sa.sohsr, sa.sop, sa.sp, sa.spqh, sa.sr, sa.st, sa.sth, sa.tr, sales.vindividualcustomer, sales.vpersondemographics, sales.vsalesperson, sales.vsalespersonsalesbyfiscalyears, sales.vsalespersonsalesbyfiscalyearsdata, sales.vstorewithaddresses, sales.vstorewithcontacts, sales.vstorewithdemographics" }, { @@ -895,6 +895,94 @@ "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "humanresources.vjobcandidate", + "Reason": "XML Functions", + "SqlStatement": "SELECT jobcandidate.jobcandidateid, jobcandidate.businessentityid, (xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Prefix\", (xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.First\", (xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Middle\", (xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Last\", (xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Suffix\", (xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"Skills\", (xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Addr.Type\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.CountryRegion\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.State\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.City\", (xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(20) AS \"Addr.PostalCode\", (xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"EMail\", (xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"WebSite\", jobcandidate.modifieddate FROM humanresources.jobcandidate", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "humanresources.vjobcandidateeducation", + "Reason": "XML Functions", + "SqlStatement": "SELECT jc.jobcandidateid, (xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Level\", (xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.StartDate\", (xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.EndDate\", (xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Degree\", (xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Major\", (xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Minor\", (xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPA\", (xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPAScale\", (xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.School\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.CountryRegion\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.State\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.City\" FROM (SELECT unnesting.jobcandidateid, CAST(('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || unnesting.education::varchar::text) || '\u003c/root\u003e'::text AS xml) AS doc FROM (SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education FROM humanresources.jobcandidate) unnesting) jc", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "humanresources.vjobcandidateemployment", + "Reason": "XML Functions", + "SqlStatement": "SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.StartDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.EndDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.OrgName\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.JobTitle\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Responsibility\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.FunctionCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.IndustryCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.CountryRegion\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.State\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.City\" FROM humanresources.jobcandidate", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "person.vadditionalcontactinfo", + "Reason": "XML Functions", + "SqlStatement": "SELECT p.businessentityid, p.firstname, p.middlename, p.lastname, (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber, btrim((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS telephonespecialinstructions, (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street, (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city, (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince, (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode, (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion, (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions, (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress, btrim((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS emailspecialinstructions, (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber, p.rowguid, p.modifieddate FROM person.person p LEFT JOIN (SELECT person.businessentityid, unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node FROM person.person WHERE person.additionalcontactinfo IS NOT NULL) additional ON p.businessentityid = additional.businessentityid", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "production.vproductmodelcatalogdescription", + "Reason": "XML Functions", + "SqlStatement": "SELECT productmodel.productmodelid, productmodel.name, (xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1]::varchar AS \"Summary\", (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar AS manufacturer, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(30) AS copyright, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS producturl, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantyperiod, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantydescription, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS noofyears, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS maintenancedescription, (xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS wheel, (xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS saddle, (xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS pedal, (xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar AS bikeframe, (xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS crankset, (xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS pictureangle, (xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS picturesize, (xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productphotoid, (xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS material, (xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS color, (xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productline, (xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS style, (xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(1024) AS riderexperience, productmodel.rowguid, productmodel.modifieddate FROM production.productmodel WHERE productmodel.catalogdescription IS NOT NULL", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "production.vproductmodelinstructions", + "Reason": "XML Functions", + "SqlStatement": "SELECT pm.productmodelid, pm.name, (xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1]::varchar AS instructions, (xpath('@LocationID'::text, pm.mfginstructions))[1]::varchar::int AS \"LocationID\", (xpath('@SetupHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"SetupHours\", (xpath('@MachineHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"MachineHours\", (xpath('@LaborHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"LaborHours\", (xpath('@LotSize'::text, pm.mfginstructions))[1]::varchar::int AS \"LotSize\", (xpath('/step/text()'::text, pm.step))[1]::varchar(1024) AS \"Step\", pm.rowguid, pm.modifieddate FROM (SELECT locations.productmodelid, locations.name, locations.rowguid, locations.modifieddate, locations.instructions, locations.mfginstructions, unnest(xpath('step'::text, locations.mfginstructions)) AS step FROM (SELECT productmodel.productmodelid, productmodel.name, productmodel.rowguid, productmodel.modifieddate, productmodel.instructions, unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions FROM production.productmodel) locations) pm", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "sales.vpersondemographics", + "Reason": "XML Functions", + "SqlStatement": "SELECT person.businessentityid, (xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::money AS totalpurchaseytd, (xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS datefirstpurchase, (xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS birthdate, (xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS maritalstatus, (xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS yearlyincome, (xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS gender, (xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS totalchildren, (xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numberchildrenathome, (xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS education, (xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS occupation, (xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::boolean AS homeownerflag, (xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numbercarsowned FROM person.person WHERE person.demographics IS NOT NULL", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "VIEW", + "ObjectName": "sales.vstorewithdemographics", + "Reason": "XML Functions", + "SqlStatement": "SELECT store.businessentityid, store.name, unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualSales\", unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualRevenue\", unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"BankName\", unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(5) AS \"BusinessType\", unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"YearOpened\", unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"Specialty\", unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"SquareFeet\", unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Brands\", unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Internet\", unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"NumberEmployees\" FROM sales.store", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" } ] } diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 9294d7ad78..1d86531c65 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -37,15 +37,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 26, + "TotalCount": 27, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 63, + "TotalCount": 64, "InvalidCount": 20, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.ordersentry, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -55,9 +55,9 @@ }, { "ObjectType": "FUNCTION", - "TotalCount": 7, - "InvalidCount": 0, - "ObjectNames": "public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.total" + "TotalCount": 9, + "InvalidCount": 2, + "ObjectNames": "public.process_order, schema2.process_order, public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.total" }, { "ObjectType": "AGGREGATE", @@ -73,9 +73,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 6, - "InvalidCount": 2, - "ObjectNames": "public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4" + "TotalCount": 7, + "InvalidCount": 3, + "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4" }, { "ObjectType": "TRIGGER", @@ -122,6 +122,7 @@ "schema2.child_table", "schema2.parent_table", "schema2.tbl_unlogged", + "public.ordersentry", "schema2.orders2", "schema2.tt", "schema2.ext_test", @@ -168,7 +169,7 @@ "test_views.abc_mview", "test_views.view_table1" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 69 objects (61 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 24 objects (5 tables/materialized views and 19 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 70 objects (62 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 24 objects (5 tables/materialized views and 19 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -228,6 +229,24 @@ "TableName": "test_xml_type", "ColumnName": "data", "DataType": "xml" + }, + { + "SchemaName": "public", + "TableName": "ordersentry_view", + "ColumnName": "order_xml", + "DataType": "xml" + }, + { + "SchemaName": "public", + "TableName": "ordersentry_view", + "ColumnName": "summary_xml", + "DataType": "xml" + }, + { + "SchemaName": "public", + "TableName": "ordersentry_view", + "ColumnName": "transaction_id", + "DataType": "xid" } ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", @@ -1007,6 +1026,20 @@ "ParentTableName": null, "SizeInBytes": 0 }, + { + "SchemaName": "public", + "ObjectName": "ordersentry", + "RowCount": 6, + "ColumnCount": 6, + "Reads": 0, + "Writes": 6, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, { "SchemaName": "schema2", "ObjectName": "tbl_unlogged", @@ -1957,5 +1990,50 @@ "Query": "SELECT xml_is_well_formed($1)", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" } + ], + "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "Advisory Locks", + "Objects": [ + { + "ObjectType": "FUNCTION", + "ObjectName": "public.process_order", + "SqlStatement": "SELECT pg_advisory_unlock(orderid);" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "schema2.process_order", + "SqlStatement": "SELECT pg_advisory_unlock(orderid);" + }, + { + "ObjectType": "VIEW", + "ObjectName": "public.ordersentry_view", + "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + }, + { + "FeatureName": "XML Functions", + "Objects": [ + { + "ObjectType": "VIEW", + "ObjectName": "public.ordersentry_view", + "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "FeatureName": "System Columns", + "Objects": [ + { + "ObjectType": "VIEW", + "ObjectName": "public.ordersentry_view", + "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + } ] } \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test/init-db b/migtests/tests/pg/assessment-report-test/init-db index dc1d41a465..07695dc83f 100755 --- a/migtests/tests/pg/assessment-report-test/init-db +++ b/migtests/tests/pg/assessment-report-test/init-db @@ -19,17 +19,19 @@ if [ ! -f "$TEMP_SQL" ]; then fi # Writing temporary SQL script to create DB objects for the test +# running the unsupported_quer_constructs.sql fileafter after all schema creation to nto get any DDL in pg_stat_statements cat < "$TEMP_SQL" \i ${TEST_DIR}/../misc-objects-1/schema.sql; \i ${TEST_DIR}/../misc-objects-2/pg_misc_objects2.sql; \i ${TEST_DIR}/../views-and-rules/pg_views_and_rules_automation.sql; \i pg_assessment_report.sql; -\i unsupported_query_constructs.sql CREATE SCHEMA IF NOT EXISTS schema2; SET SEARCH_PATH TO schema2; \i ${TEST_DIR}/../misc-objects-1/schema.sql; \i ${TEST_DIR}/../misc-objects-2/pg_misc_objects2.sql; \i pg_assessment_report.sql; +SET SEARCH_PATH TO public; +\i unsupported_query_constructs.sql; EOF # Run the temporary SQL script diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index da73b1ceaa..fec2cb1edf 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -222,3 +222,65 @@ CREATE TRIGGER before_sales_region_insert_update BEFORE INSERT OR UPDATE ON public.sales_region FOR EACH ROW EXECUTE FUNCTION public.check_sales_region(); + + + CREATE TABLE public.ordersentry ( + order_id SERIAL PRIMARY KEY, + customer_name TEXT NOT NULL, + product_name TEXT NOT NULL, + quantity INT NOT NULL, + price NUMERIC(10, 2) NOT NULL, + processed_at timestamp +); + +INSERT INTO public.ordersentry (customer_name, product_name, quantity, price) +VALUES + ('Alice', 'Laptop', 1, 1200.00), + ('Bob', 'Smartphone', 2, 800.00), + ('Charlie', 'Tablet', 1, 500.00); + +CREATE VIEW public.ordersentry_view AS +SELECT + order_id, + customer_name, + product_name, + quantity, + price, + xmlelement( + name "OrderDetails", + xmlelement(name "Customer", customer_name), + xmlelement(name "Product", product_name), + xmlelement(name "Quantity", quantity), + xmlelement(name "TotalPrice", price * quantity) + ) AS order_xml, + xmlconcat( + xmlelement(name "Customer", customer_name), + xmlelement(name "Product", product_name) + ) AS summary_xml, + pg_try_advisory_lock(hashtext(customer_name || product_name)) AS lock_acquired, + ctid AS row_ctid, + xmin AS transaction_id +FROM + ordersentry; + +CREATE OR REPLACE FUNCTION process_order(orderid INT) RETURNS VOID AS $$ +DECLARE + lock_acquired BOOLEAN; +BEGIN + lock_acquired := pg_try_advisory_lock(orderid); -- not able to report this as it is an assignment statement TODO: fix when support this + + IF NOT lock_acquired THEN + RAISE EXCEPTION 'Order % already being processed by another session', orderid; + END IF; + + UPDATE orders + SET processed_at = NOW() + WHERE orders.order_id = orderid; + + RAISE NOTICE 'Order % processed successfully', orderid; + + PERFORM pg_advisory_unlock(orderid); +END; +$$ LANGUAGE plpgsql; + +select process_order(1); diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 6474fcb025..d2151ea2f8 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -13508,5 +13508,6 @@ "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index b82d09d963..17a32e2916 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -5697,5 +5697,6 @@ "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 8ee648b9c1..783a2ed7b7 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -313,5 +313,6 @@ "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index f14bd0b852..ba92bad2b4 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -235,5 +235,6 @@ "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 275de27adb..dd13587bec 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -26597,5 +26597,6 @@ "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index 14301a9481..092ca5739b 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -963,5 +963,6 @@ "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 7a73767908..fbbfa6cadd 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -363,5 +363,6 @@ "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 5bd7fd9efd..1c6f8dd993 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -1272,5 +1272,6 @@ "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." } ], - "UnsupportedQueryConstructs": null + "UnsupportedQueryConstructs": null, + "UnsupportedPlPgSqlObjects": null } diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 451e32971a..cd1f9a345b 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1604,6 +1604,7 @@ func checkPlPgSQLStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType st } func convertIssueInstanceToAnalyzeIssue(issueInstance issue.IssueInstance, fileName string) utils.Issue { + summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true return utils.Issue{ ObjectType: issueInstance.ObjectType, ObjectName: issueInstance.ObjectName, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 0fd1bb8199..38de5af76c 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -155,6 +155,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { return callhome.UnsupportedFeature{ FeatureName: feature.FeatureName, ObjectCount: len(feature.Objects), + TotalOccurrences: len(feature.Objects), } })), UnsupportedQueryConstructs: callhome.MarshalledJsonString(countByConstructType), @@ -163,6 +164,15 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { return callhome.UnsupportedFeature{ FeatureName: feature.FeatureName, ObjectCount: len(feature.Objects), + TotalOccurrences: len(feature.Objects), + } + })), + UnsupportedPlPgSqlObjects: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedPlPgSqlObjects, func(plpgsql UnsupportedFeature, _ int) callhome.UnsupportedFeature { + groupedObjects := groupByObjectName(plpgsql.Objects) + return callhome.UnsupportedFeature{ + FeatureName: plpgsql.FeatureName, + ObjectCount: len(lo.Keys(groupedObjects)), + TotalOccurrences: len(plpgsql.Objects), } })), TableSizingStats: callhome.MarshalledJsonString(tableSizingStats), @@ -524,6 +534,19 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment }) } + for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { + for _, object := range plpgsqlObjects.Objects { + issues = append(issues, AssessmentIssuePayload{ + Type: PLPGSQL_OBJECT, + TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, + Subtype: plpgsqlObjects.FeatureName, + SubtypeDescription: plpgsqlObjects.FeatureDescription, + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: plpgsqlObjects.DocsLink, + }) + } + } return issues } @@ -857,7 +880,7 @@ func getAssessmentReportContentFromAnalyzeSchema() error { } assessmentReport.UnsupportedFeatures = append(assessmentReport.UnsupportedFeatures, unsupportedFeatures...) assessmentReport.UnsupportedFeaturesDesc = FEATURE_ISSUE_TYPE_DESCRIPTION - if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", false) { + if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { unsupportedPlpgSqlObjects := fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport) assessmentReport.UnsupportedPlPgSqlObjects = unsupportedPlpgSqlObjects } @@ -1288,11 +1311,11 @@ func generateAssessmentReportHtml(reportDir string) error { log.Infof("creating template for assessment report...") funcMap := template.FuncMap{ - "split": split, - "groupByObjectType": groupByObjectType, - "numKeysInMapStringObjectInfo": numKeysInMapStringObjectInfo, - "groupByObjectName": groupByObjectName, - "totalUniqueObjectNamesOfAllTypes": totalUniqueObjectNamesOfAllTypes, + "split": split, + "groupByObjectType": groupByObjectType, + "numKeysInMapStringObjectInfo": numKeysInMapStringObjectInfo, + "groupByObjectName": groupByObjectName, + "totalUniqueObjectNamesOfAllTypes": totalUniqueObjectNamesOfAllTypes, } tmpl := template.Must(template.New("report").Funcs(funcMap).Parse(string(bytesTemplate))) diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 6037158f9c..112e6fdeb2 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1125,7 +1125,7 @@ type AssessmentReport struct { UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` - UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects,omitempty"` + UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` Notes []string `json:"Notes"` @@ -1140,7 +1140,7 @@ type UnsupportedFeature struct { } type ObjectInfo struct { - ObjectType string `json:"-"` //only using as of now for html report Unsupported PLpg/SQL objects feature + ObjectType string `json:"ObjectType,omitempty"` ObjectName string SqlStatement string } diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 9eeaa4895a..262e4a0b35 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -174,12 +174,14 @@ const ( DATATYPE = "datatype" QUERY_CONSTRUCT = "query_construct" // confused: in json for some values we are using space separated and for some snake_case MIGRATION_CAVEATS = "migration_caveats" + PLPGSQL_OBJECT = "plpgsql_object" // Description FEATURE_ISSUE_TYPE_DESCRIPTION = "Features of the source database that are not supported on the target YugabyteDB." DATATYPE_ISSUE_TYPE_DESCRIPTION = "Data types of the source database that are not supported on the target YugabyteDB." MIGRATION_CAVEATS_TYPE_DESCRIPTION = "Migration Caveats highlights the current limitations with the migration workflow." UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION = "Source database queries not supported in YugabyteDB, identified by scanning system tables." + UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PLPGSQL code block" SCHEMA_SUMMARY_DESCRIPTION = "Objects that will be created on the target YugabyteDB." SCHEMA_SUMMARY_DESCRIPTION_ORACLE = SCHEMA_SUMMARY_DESCRIPTION + " Some of the index and sequence names might be different from those in the source database." diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 10cfbbbefa..29e032eaec 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -92,8 +92,9 @@ type TargetDBDetails struct { } type UnsupportedFeature struct { - FeatureName string `json:"FeatureName"` - ObjectCount int `json:"ObjectCount"` + FeatureName string `json:"FeatureName"` + ObjectCount int `json:"ObjectCount"` + TotalOccurrences int `json:"TotalOccurrences"` } type AssessMigrationPhasePayload struct { @@ -102,6 +103,7 @@ type AssessMigrationPhasePayload struct { UnsupportedDatatypes string `json:"unsupported_datatypes"` UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` MigrationCaveats string `json:"migration_caveats"` + UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` Error string `json:"error,omitempty"` // Removed it for now, TODO TableSizingStats string `json:"table_sizing_stats"` IndexSizingStats string `json:"index_sizing_stats"` diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 8b12140f4c..9b9ca22a89 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -43,7 +43,7 @@ func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, er return nil, fmt.Errorf("error parsing query: %w", err) } if queryparser.IsPLPGSQLObject(parseTree) { - plpgsqlObjType, plpgsqlObjName := queryparser.GetObjectTypeAndObjectName(parseTree) + objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) plpgsqlQueries, err := queryparser.GetAllPLPGSQLStatements(query) if err != nil { return nil, fmt.Errorf("error getting all the queries from query: %w", err) @@ -56,17 +56,36 @@ func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, er log.Errorf("error getting issues in query-%s: %v", query, err) continue } - for _, i := range issuesInQuery { - //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object - //e.g. replacing the DML_QUERY and "" to FUNCTION and - i.ObjectType = plpgsqlObjType - i.ObjectName = plpgsqlObjName - issues = append(issues, i) - } + issues = append(issues, issuesInQuery...) + } + return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { + //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object + //e.g. replacing the DML_QUERY and "" to FUNCTION and + i.ObjectType = objType + i.ObjectName = objName + return i + }), nil + } + //Handle the Mview/View DDL's Select stmt issues + if queryparser.IsViewObject(parseTree) || queryparser.IsMviewObject(parseTree) { + objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) + selectStmtQuery, err := queryparser.GetSelectStmtQueryFromViewOrMView(parseTree) + if err != nil { + return nil, fmt.Errorf("error deparsing a select stmt: %v", err) } - return issues, nil + issues, err := p.GetIssues(selectStmtQuery) + if err != nil { + return nil, err + } + return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { + //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object + //e.g. replacing the DML_QUERY and "" to FUNCTION and + i.ObjectType = objType + i.ObjectName = objName + return i + }), nil + } - //TODO: add handling for VIEW/MVIEW to parse select return p.getDMLIssues(query, parseTree) } diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/queryparser/query_parser.go index 1b735814b5..8e0699d499 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/queryparser/query_parser.go @@ -42,6 +42,25 @@ func ParsePLPGSQLToJson(query string) (string, error) { return jsonString, err } +func DeparseSelectStmt(selectStmt *pg_query.SelectStmt) (string, error) { + if selectStmt != nil { + parseResult := &pg_query.ParseResult{ + Stmts: []*pg_query.RawStmt{ + { + Stmt: &pg_query.Node{ + Node: &pg_query.Node_SelectStmt{SelectStmt: selectStmt}, + }, + }, + }, + } + + // Deparse the SelectStmt to get the string representation + selectSQL, err := pg_query.Deparse(parseResult) + return selectSQL, err + } + return "", nil +} + func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { return parseTree.Stmts[0].Stmt.ProtoReflect() } @@ -52,8 +71,36 @@ func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { return isPlPgSQLObject } +func IsViewObject(parseTree *pg_query.ParseResult) bool { + _, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) + return isViewStmt +} + +func IsMviewObject(parseTree *pg_query.ParseResult) bool { + createAsNode, isCreateAsStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) //for MVIEW case + return isCreateAsStmt && createAsNode.CreateTableAsStmt.Objtype == pg_query.ObjectType_OBJECT_MATVIEW +} + +func GetSelectStmtQueryFromViewOrMView(parseTree *pg_query.ParseResult) (string, error) { + viewNode, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) + createAsNode, _ := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt)//For MVIEW case + var selectStmt *pg_query.SelectStmt + if isViewStmt { + selectStmt = viewNode.ViewStmt.GetQuery().GetSelectStmt() + } else { + selectStmt = createAsNode.CreateTableAsStmt.GetQuery().GetSelectStmt() + } + selectStmtQuery, err := DeparseSelectStmt(selectStmt) + if err != nil { + return "", fmt.Errorf("deparsing the select stmt: %v", err) + } + return selectStmtQuery, nil +} + func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { createFuncNode, isCreateFunc := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) + viewNode, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) + createAsNode, _ := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) switch true { case isCreateFunc: /* @@ -72,8 +119,22 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string } funcNameList := stmt.GetFuncname() return objectType, getFunctionObjectName(funcNameList) + case isViewStmt: + viewName := viewNode.ViewStmt.View + return "VIEW", getObjectNameFromRangeVar(viewName) + case IsMviewObject(parseTree): + intoMview := createAsNode.CreateTableAsStmt.Into.Rel + return "MVIEW", getObjectNameFromRangeVar(intoMview) + default: + panic("unsupported type of parseResult") } - return "", "" +} + +// Range Var is the struct to get the relation information like relation name, schema name, persisted relation or not, etc.. +func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string { + schema := obj.Schemaname + name := obj.Relname + return lo.Ternary(schema != "", fmt.Sprintf("%s.%s", schema, name), name) } func getFunctionObjectName(funcNameList []*pg_query.Node) string { From b61b8099f47aa3af4b113ee44c712d81fcdaec71 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Wed, 20 Nov 2024 16:26:45 +0530 Subject: [PATCH 005/105] YBVersion to be able to store and compare yb versions (#1901) YBVersion is a wrapper around hashicorp/go-version.Version that adds some Yugabyte-specific functionality. - It only supports versions with 2, 3, or 4 segments (A.B, A.B.C, A.B.C.D). - It only accepts one of supported Yugabyte version series. - It provides a method to compare versions based on common prefix of segments. This is useful in cases where the version is not fully specified (e.g. 2.20 vs 2.20.7.0). - If the user specifies 2.20, a reasonable assumption is that they would be on the latest 2.20.x.y version. Therefore, we want only want to compare the prefix 2.20 between versions and ignore the rest. --- yb-voyager/go.mod | 1 + yb-voyager/go.sum | 2 + yb-voyager/src/version/yb_version.go | 159 ++++++++++++++++++++++ yb-voyager/src/version/yb_version_test.go | 122 +++++++++++++++++ 4 files changed, 284 insertions(+) create mode 100644 yb-voyager/src/version/yb_version.go create mode 100644 yb-voyager/src/version/yb_version_test.go diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 34b3af391c..8f4c858131 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -42,6 +42,7 @@ require ( require ( github.com/fergusstrange/embedded-postgres v1.29.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/lib/pq v1.10.9 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index e948346735..58e148305e 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1260,6 +1260,8 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/yb-voyager/src/version/yb_version.go b/yb-voyager/src/version/yb_version.go new file mode 100644 index 0000000000..e5ffd7d14a --- /dev/null +++ b/yb-voyager/src/version/yb_version.go @@ -0,0 +1,159 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strconv" + "strings" + + "github.com/hashicorp/go-version" + "github.com/samber/lo" + "golang.org/x/exp/slices" +) + +// Reference - https://docs.yugabyte.com/preview/releases/ybdb-releases/ +var supportedYBVersionStableSeriesOld = []string{"2.14", "2.18", "2.20"} +var supportedYBVersionStableSeries = []string{"2024.1"} +var supportedYBVersionPreviewSeries = []string{"2.21", "2.23"} + +var allSupportedYBVersionSeries = lo.Flatten([][]string{supportedYBVersionStableSeries, supportedYBVersionPreviewSeries, supportedYBVersionStableSeriesOld}) + +const ( + STABLE = "stable" + PREVIEW = "preview" + STABLE_OLD = "stable_old" +) + +/* +YBVersion is a wrapper around hashicorp/go-version.Version that adds some Yugabyte-specific +functionality. + 1. It only supports versions with 2, 3, or 4 segments (A.B, A.B.C, A.B.C.D). + 2. It only accepts one of supported Yugabyte version series. + 3. It provides a method to compare versions based on common prefix of segments. This is useful in cases + where the version is not fully specified (e.g. 2.20 vs 2.20.7.0). + If the user specifies 2.20, a reasonable assumption is that they would be on the latest 2.20.x.y version. + Therefore, we want only want to compare the prefix 2.20 between versions and ignore the rest. +*/ +type YBVersion struct { + *version.Version +} + +func NewYBVersion(v string) (*YBVersion, error) { + v1, err := version.NewVersion(v) + if err != nil { + return nil, err + } + + ybv := &YBVersion{v1} + origSegLen := ybv.originalSegmentsLen() + if origSegLen < 2 || origSegLen > 4 { + return nil, fmt.Errorf("invalid YB version: %s. Version should have between min 2 and max 4 segments (A.B.C.D). Version %s has ybv.originalSegmentsLen()", v, v) + } + + if !slices.Contains(allSupportedYBVersionSeries, ybv.Series()) { + return nil, fmt.Errorf("unsupported YB version series: %s. Supported YB version series = %v", ybv.Series(), allSupportedYBVersionSeries) + } + return ybv, nil +} + +// The first two segments essentially represent the release series +// as per https://docs.yugabyte.com/preview/releases/ybdb-releases/ +func (ybv *YBVersion) Series() string { + return joinIntsWith(ybv.Segments()[:2], ".") +} + +func (ybv *YBVersion) ReleaseType() string { + series := ybv.Series() + if slices.Contains(supportedYBVersionStableSeries, series) { + return STABLE + } else if slices.Contains(supportedYBVersionPreviewSeries, series) { + return PREVIEW + } else if slices.Contains(supportedYBVersionStableSeriesOld, series) { + return STABLE_OLD + } else { + panic("unknown release type for series: " + series) + } +} + +// This returns the len of the segments in the original +// input. For instance if input is 2024.1, +// go-version.Version.Segments() will return [2024, 1, 0, 0] +// original = 2024.1 +// originalSegmentsLen = 2 ([2024,1]) +func (ybv *YBVersion) originalSegmentsLen() int { + orig := ybv.Original() + segments := strings.Split(orig, ".") + return len(segments) +} + +/* +Compare the common prefix of segments between two versions. +Similar to method go-version.Version.Compare, but only compares the common prefix of segments. +This is useful in cases where the version is not fully specified (e.g. 2.20 vs 2.20.7.0). +If the user wants to compare 2.20 and 2.20.7.0, a reasonable assumption is that they would be on the latest 2.20.x.y version. +Therefore, we want only want to compare the prefix 2.20 between versions and ignore the rest. + +This returns -1, 0, or 1 if this version is smaller, equal, +or larger than the other version, respectively. +*/ +func (ybv *YBVersion) CompareCommonPrefix(other *YBVersion) (int, error) { + if ybv.Series() != other.Series() { + return 0, fmt.Errorf("cannot compare versions with different series: %s and %s", ybv.Series(), other.Series()) + } + myOriginalSegLen := ybv.originalSegmentsLen() + otherOriginalSegLen := other.originalSegmentsLen() + minSegLen := min(myOriginalSegLen, otherOriginalSegLen) + + ybvMin, err := version.NewVersion(joinIntsWith(ybv.Segments()[:minSegLen], ".")) + if err != nil { + return 0, fmt.Errorf("create version from segments: %v", ybv.Segments()[:minSegLen]) + } + otherMin, err := version.NewVersion(joinIntsWith(other.Segments()[:minSegLen], ".")) + if err != nil { + return 0, fmt.Errorf("create version from segments: %v", other.Segments()[:minSegLen]) + } + return ybvMin.Compare(otherMin), nil +} + +func (ybv *YBVersion) CommonPrefixGreaterThanOrEqual(other *YBVersion) (bool, error) { + res, err := ybv.CompareCommonPrefix(other) + if err != nil { + return false, err + } + return res >= 0, nil +} + +func (ybv *YBVersion) CommonPrefixLessThan(other *YBVersion) (bool, error) { + res, err := ybv.CompareCommonPrefix(other) + if err != nil { + return false, err + } + return res < 0, nil +} + +func (ybv *YBVersion) String() string { + return ybv.Original() +} + +func joinIntsWith(ints []int, delimiter string) string { + strs := make([]string, len(ints)) + for i, v := range ints { + strs[i] = strconv.Itoa(v) + } + return strings.Join(strs, delimiter) +} diff --git a/yb-voyager/src/version/yb_version_test.go b/yb-voyager/src/version/yb_version_test.go new file mode 100644 index 0000000000..024d991ab1 --- /dev/null +++ b/yb-voyager/src/version/yb_version_test.go @@ -0,0 +1,122 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidNewYBVersion(t *testing.T) { + validVersionStrings := []string{ + "2024.1.1", + "2.20", + "2.21.2.1", + } + for _, v := range validVersionStrings { + _, err := NewYBVersion(v) + assert.NoError(t, err) + } +} + +func TestInvalidNewYBVersion(t *testing.T) { + invalidVersionStrings := []string{ + "abc.def", // has to be numbers + "2024.0.1-1", // has to be in supported series + "2024", // has to have at least 2 segments + "2024.1.1.1.1.1", // max 4 segments + } + for _, v := range invalidVersionStrings { + _, err := NewYBVersion(v) + assert.Error(t, err) + } +} + +func TestStableReleaseType(t *testing.T) { + stableVersionStrings := []string{ + "2024.1.1", + "2024.1", + "2024.1.1.1", + } + for _, v := range stableVersionStrings { + ybVersion, _ := NewYBVersion(v) + assert.Equal(t, STABLE, ybVersion.ReleaseType()) + } +} + +func TestPreviewReleaseType(t *testing.T) { + previewVersionStrings := []string{ + "2.21.1", + "2.21", + } + for _, v := range previewVersionStrings { + ybVersion, _ := NewYBVersion(v) + assert.Equal(t, PREVIEW, ybVersion.ReleaseType()) + } +} + +func TestStableOldReleaseType(t *testing.T) { + stableOldVersionStrings := []string{ + "2.20.1", + "2.20", + } + for _, v := range stableOldVersionStrings { + ybVersion, _ := NewYBVersion(v) + assert.Equal(t, STABLE_OLD, ybVersion.ReleaseType()) + } +} + +func TestVersionPrefixGreaterThanOrEqual(t *testing.T) { + versionsToCompare := [][]string{ + {"2024.1.1", "2024.1.0"}, + {"2024.1", "2024.1.4.0"}, //prefix 2024.1 == 2024.1 + } + for _, v := range versionsToCompare { + v1, err := NewYBVersion(v[0]) + assert.NoError(t, err) + v2, err := NewYBVersion(v[1]) + assert.NoError(t, err) + greaterThan, err := v1.CommonPrefixGreaterThanOrEqual(v2) + assert.NoError(t, err) + assert.True(t, greaterThan, "%s >= %s", v[0], v[1]) + } +} + +func TestVersionPrefixLessThan(t *testing.T) { + versionsToCompare := [][]string{ + {"2024.1.0", "2024.1.2"}, + {"2.21.1", "2.21.7"}, + {"2.20.1", "2.20.7"}, + } + for _, v := range versionsToCompare { + v1, err := NewYBVersion(v[0]) + assert.NoError(t, err) + v2, err := NewYBVersion(v[1]) + assert.NoError(t, err) + lessThan, err := v1.CommonPrefixLessThan(v2) + assert.NoError(t, err) + assert.True(t, lessThan, "%s < %s", v[0], v[1]) + } +} + +func TestComparingDifferentSeriesThrowsError(t *testing.T) { + v1, _ := NewYBVersion("2024.1.1") + v2, _ := NewYBVersion("2.21.1") + _, err := v1.CompareCommonPrefix(v2) + assert.Error(t, err) +} From 9ba217745b0eac0f5379ae7b8724cba9186dfcce Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 20 Nov 2024 18:12:28 +0530 Subject: [PATCH 006/105] Send objectName for Extension Issue / Unsupported feature to callhome in analyze/assess payload (#1910) --- yb-voyager/cmd/analyzeSchema.go | 18 ++++++++++++++---- yb-voyager/cmd/assessMigrationCommand.go | 23 ++++++++++++++++++----- yb-voyager/src/callhome/diagnostics.go | 13 ++++++++++--- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index cd1f9a345b..a422e58a3c 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -2134,7 +2134,8 @@ func generateAnalyzeSchemaReport(msr *metadb.MigrationStatusRecord, reportFormat return nil } -var reasonsIncludingSensitiveInformation = []string{ +// analyze issue reasons to modify the reason before sending to callhome as will have sensitive information +var reasonsIncludingSensitiveInformationToCallhome = []string{ UNSUPPORTED_PG_SYNTAX, POLICY_ROLE_ISSUE, UNSUPPORTED_DATATYPE, @@ -2144,6 +2145,11 @@ var reasonsIncludingSensitiveInformation = []string{ INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, } +// analyze issue reasons to send the object names for to callhome +var reasonsToSendObjectNameToCallhome = []string{ + UNSUPPORTED_EXTENSION_ISSUE, +} + func packAndSendAnalyzeSchemaPayload(status string) { if !shouldSendCallhome() { return @@ -2153,9 +2159,13 @@ func packAndSendAnalyzeSchemaPayload(status string) { payload.MigrationPhase = ANALYZE_PHASE var callhomeIssues []utils.Issue for _, issue := range schemaAnalysisReport.Issues { - issue.SqlStatement = "" // Obfuscate sensitive information before sending to callhome cluster - issue.ObjectName = "XXX" // Redacting object name before sending - for _, sensitiveReason := range reasonsIncludingSensitiveInformation { + issue.SqlStatement = "" // Obfuscate sensitive information before sending to callhome cluster + if !lo.ContainsBy(reasonsToSendObjectNameToCallhome, func(r string) bool { + return strings.Contains(issue.Reason, r) + }) { + issue.ObjectName = "XXX" // Redacting object name before sending in case reason is not in list + } + for _, sensitiveReason := range reasonsIncludingSensitiveInformationToCallhome { if strings.Contains(issue.Reason, sensitiveReason) { switch sensitiveReason { case UNSUPPORTED_DATATYPE, UNSUPPORTED_DATATYPE_LIVE_MIGRATION: diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 38de5af76c..b69484a96b 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -105,6 +105,11 @@ var assessMigrationCmd = &cobra.Command{ }, } +// Assessment feature names to send the object names for to callhome +var featuresToSendObjectsToCallhome = []string{ + EXTENSION_FEATURE, +} + func packAndSendAssessMigrationPayload(status string, errMsg string) { if !shouldSendCallhome() { return @@ -152,18 +157,26 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { assessPayload := callhome.AssessMigrationPhasePayload{ MigrationComplexity: assessmentReport.MigrationComplexity, UnsupportedFeatures: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedFeatures, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature { - return callhome.UnsupportedFeature{ - FeatureName: feature.FeatureName, - ObjectCount: len(feature.Objects), + var objects []string + if slices.Contains(featuresToSendObjectsToCallhome, feature.FeatureName) { + objects = lo.Map(feature.Objects, func(o ObjectInfo, _ int) string { + return o.ObjectName + }) + } + res := callhome.UnsupportedFeature{ + FeatureName: feature.FeatureName, + ObjectCount: len(feature.Objects), + Objects: objects, TotalOccurrences: len(feature.Objects), } + return res })), UnsupportedQueryConstructs: callhome.MarshalledJsonString(countByConstructType), UnsupportedDatatypes: callhome.MarshalledJsonString(unsupportedDatatypesList), MigrationCaveats: callhome.MarshalledJsonString(lo.Map(assessmentReport.MigrationCaveats, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature { return callhome.UnsupportedFeature{ - FeatureName: feature.FeatureName, - ObjectCount: len(feature.Objects), + FeatureName: feature.FeatureName, + ObjectCount: len(feature.Objects), TotalOccurrences: len(feature.Objects), } })), diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 29e032eaec..5b65b609f0 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -76,6 +76,8 @@ type Payload struct { Status string `json:"status"` } +// SHOULD NOT REMOVE THESE (host, db_type, db_version, total_db_size_bytes) FIELDS of SourceDBDetails as parsing these specifically here +// https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L549 type SourceDBDetails struct { Host string `json:"host"` //keeping it empty for now, as field is parsed in big query app DBType string `json:"db_type"` @@ -84,6 +86,8 @@ type SourceDBDetails struct { Role string `json:"role,omitempty"` //for differentiating replica details } +// SHOULD NOT REMOVE THESE (host, db_version, node_count, total_cores) FIELDS of TargetDBDetails as parsing these specifically here +// https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L556 type TargetDBDetails struct { Host string `json:"host"` DBVersion string `json:"db_version"` @@ -92,9 +96,10 @@ type TargetDBDetails struct { } type UnsupportedFeature struct { - FeatureName string `json:"FeatureName"` - ObjectCount int `json:"ObjectCount"` - TotalOccurrences int `json:"TotalOccurrences"` + FeatureName string `json:"FeatureName"` + Objects []string `json:"Objects,omitempty"` + ObjectCount int `json:"ObjectCount"` + TotalOccurrences int `json:"TotalOccurrences"` } type AssessMigrationPhasePayload struct { @@ -131,6 +136,8 @@ type ExportSchemaPhasePayload struct { CommentsOnObjects bool `json:"comments_on_objects"` } +// SHOULD NOT REMOVE THESE TWO (issues, database_objects) FIELDS of AnalyzePhasePayload as parsing these specifically here +// https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L563 type AnalyzePhasePayload struct { Issues string `json:"issues"` DatabaseObjects string `json:"database_objects"` From 50585a3028ae094d7b1a2ab2b997b63b7e34ff1e Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Wed, 20 Nov 2024 20:57:52 +0530 Subject: [PATCH 007/105] Removed superuser from Fall Back automation (#1894) --- migtests/scripts/functions.sh | 127 +++++++++++++----- .../scripts/live-migration-fallb-run-test.sh | 3 +- 2 files changed, 99 insertions(+), 31 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 2b48f9f5b5..9059bea7c4 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -532,37 +532,38 @@ get_data_migration_report(){ } verify_report() { - expected_report=$1 - actual_report=$2 - if [ -f "${actual_report}" ] - then - echo "Printing ${actual_report} file" - cat "${actual_report}" - # Parse JSON data - actual_data=$(jq -c '.' "${actual_report}") + expected_report=$1 + actual_report=$2 + + if [ -f "${actual_report}" ]; then + # Parse and sort JSON data + actual_data=$(jq -c '.' "${actual_report}" | jq -S 'sort_by(.table_name)') - if [ -f "${expected_report}" ] - then - expected_data=$(jq -c '.' "${expected_report}") + if [ -f "${expected_report}" ]; then + expected_data=$(jq -c '.' "${expected_report}" | jq -S 'sort_by(.table_name)') + + # Save the sorted JSON data to temporary files + temp_actual=$(mktemp) + temp_expected=$(mktemp) + echo "$actual_data" > "$temp_actual" + echo "$expected_data" > "$temp_expected" + + compare_files "$temp_actual" "$temp_expected" - # Compare data - actual_data=$(echo $actual_data | jq -S 'sort_by(.table_name)') - expected_data=$(echo $expected_data | jq -S 'sort_by(.table_name)') - if [ "$actual_data" == "$expected_data" ] - then - echo "Data matches expected report." - else - echo "Data does not match expected report." - exit 1 + # Clean up temporary files + rm "$temp_actual" "$temp_expected" + + # If files do not match, exit + if [ $? -ne 0 ]; then + exit 1 fi else echo "No ${expected_report} found." - # exit 1 fi - else - echo "No ${actual_report} found." - exit 1 - fi + else + echo "No ${actual_report} found." + exit 1 + fi } @@ -678,14 +679,80 @@ setup_fallback_environment() { if [ "${SOURCE_DB_TYPE}" = "oracle" ]; then run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql - elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then - cat > alter_user_superuser.sql < "${disable_triggers_sql}" +DO \$\$ +DECLARE + r RECORD; +BEGIN + FOR r IN + SELECT table_schema, '"' || table_name || '"' AS t_name + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + AND table_schema IN ('${formatted_schema_list}') + LOOP + EXECUTE 'ALTER TABLE ' || r.table_schema || '.' || r.t_name || ' DISABLE TRIGGER ALL'; + END LOOP; +END \$\$; EOF - run_psql ${SOURCE_DB_NAME} "$(cat alter_user_superuser.sql)" - + + # Dropping Fkeys + cat < "${drop_constraints_sql}" +DO \$\$ +DECLARE + fk RECORD; +BEGIN + FOR fk IN + SELECT conname, conrelid::regclass AS table_name + FROM pg_constraint + JOIN pg_class ON conrelid = pg_class.oid + JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace + WHERE contype = 'f' + AND pg_namespace.nspname IN ('${formatted_schema_list}') + LOOP + EXECUTE 'ALTER TABLE ' || fk.table_name || ' DROP CONSTRAINT ' || fk.conname; + END LOOP; +END \$\$; +EOF + + psql_import_file "${SOURCE_DB_NAME}" "${disable_triggers_sql}" + psql_import_file "${SOURCE_DB_NAME}" "${drop_constraints_sql}" + + rm -f "${disable_triggers_sql}" "${drop_constraints_sql}" fi +} +reenable_triggers_fkeys() { + if [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then + enable_triggers_sql=$(mktemp) + formatted_schema_list=$(echo "${SOURCE_DB_SCHEMA}" | sed "s/,/','/g") + + cat < "${enable_triggers_sql}" +DO \$\$ +DECLARE + r RECORD; +BEGIN + FOR r IN + SELECT table_schema, '"' || table_name || '"' AS t_name + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + AND table_schema IN ('${formatted_schema_list}') + LOOP + EXECUTE 'ALTER TABLE ' || r.table_schema || '.' || r.t_name || ' ENABLE TRIGGER ALL'; + END LOOP; +END \$\$; +EOF + psql_import_file "${SOURCE_DB_NAME}" "${enable_triggers_sql}" + fi +#TODO: Add re-creating FKs } assess_migration() { diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index 1ddf5221cd..fc4fb886a5 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -236,7 +236,8 @@ main() { run_ysql ${TARGET_DB_NAME} "\di" run_ysql ${TARGET_DB_NAME} "\dft" - + step "Re-Enable Triggers and Fkeys" + reenable_triggers_fkeys step "Run final validations." if [ -x "${TEST_DIR}/validateAfterChanges" ] From 2dc120a626b23aa0df4fb0d1d0b52edf93b607da Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:08:01 +0530 Subject: [PATCH 008/105] Doing some initial changes for full table list integration (#1824) --- yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/cmd/exportData.go | 63 +++++++++++++++++------- yb-voyager/cmd/exportSchema.go | 1 + yb-voyager/src/namereg/namereg.go | 4 +- yb-voyager/src/srcdb/mysql.go | 4 ++ yb-voyager/src/srcdb/oracle.go | 4 ++ yb-voyager/src/srcdb/postgres.go | 30 +---------- yb-voyager/src/srcdb/srcdb.go | 1 + yb-voyager/src/srcdb/yugabytedb.go | 4 ++ 9 files changed, 64 insertions(+), 48 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index b69484a96b..6930a1b29d 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -335,6 +335,7 @@ func assessMigration() (err error) { // Check if source db has permissions to assess migration if source.RunGuardrailsChecks { + checkIfSchemasHaveUsagePermissions() missingPerms, err := source.DB().GetMissingExportSchemaPermissions() if err != nil { return fmt.Errorf("failed to get missing assess migration permissions: %w", err) diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index 46522c0342..6040cf53ea 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -239,9 +239,8 @@ func exportData() bool { utils.ErrExit("schema %q does not exist", source.Schema) } - // Check if source DB has required permissions for export data if source.RunGuardrailsChecks { - checkExportDataPermissions() + checkIfSchemasHaveUsagePermissions() } clearMigrationStateIfRequired() @@ -257,7 +256,16 @@ func exportData() bool { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var partitionsToRootTableMap map[string]string - partitionsToRootTableMap, finalTableList, tablesColumnList := getFinalTableColumnList() + // get initial table list + partitionsToRootTableMap, finalTableList := getInitialTableList() + + // Check if source DB has required permissions for export data + if source.RunGuardrailsChecks { + checkExportDataPermissions() + } + + // finalize table list and column list + finalTableList, tablesColumnList := finalizeTableColumnList(finalTableList) if len(finalTableList) == 0 { utils.PrintAndLog("no tables present to export, exiting...") @@ -483,6 +491,24 @@ func checkExportDataPermissions() { } } +func checkIfSchemasHaveUsagePermissions() { + schemasMissingUsage, err := source.DB().GetSchemasMissingUsagePermissions() + if err != nil { + utils.ErrExit("get schemas missing usage permissions: %v", err) + } + if len(schemasMissingUsage) > 0 { + utils.PrintAndLog("\n%s[%s]", color.RedString(fmt.Sprintf("Missing USAGE permission for user %s on Schemas: ", source.User)), strings.Join(schemasMissingUsage, ", ")) + + var link string + if changeStreamingIsEnabled(exportType) { + link = "https://docs.yugabyte.com/preview/yugabyte-voyager/migrate/live-migrate/#prepare-the-source-database" + } else { + link = "https://docs.yugabyte.com/preview/yugabyte-voyager/migrate/migrate-steps/#prepare-the-source-database" + } + utils.ErrExit("\nCheck the documentation to prepare the database for migration: %s", color.BlueString(link)) + } +} + func updateCallhomeExportPhase() { if !callhome.SendDiagnostics { return @@ -713,10 +739,10 @@ func reportUnsupportedTables(finalTableList []sqlname.NameTuple) { } } -func getFinalTableColumnList() (map[string]string, []sqlname.NameTuple, *utils.StructMap[sqlname.NameTuple, []string]) { +func getInitialTableList() (map[string]string, []sqlname.NameTuple) { var tableList []sqlname.NameTuple // store table list after filtering unsupported or unnecessary tables - var finalTableList, skippedTableList []sqlname.NameTuple + var finalTableList []sqlname.NameTuple tableListFromDB := source.DB().GetAllTableNames() var err error var fullTableList []sqlname.NameTuple @@ -766,11 +792,23 @@ func getFinalTableColumnList() (map[string]string, []sqlname.NameTuple, *utils.S } }) } + var partitionsToRootTableMap map[string]string + isTableListSet := source.TableList != "" + partitionsToRootTableMap, finalTableList, err = addLeafPartitionsInTableList(finalTableList, isTableListSet) + if err != nil { + utils.ErrExit("failed to add the leaf partitions in table list: %w", err) + } + + return partitionsToRootTableMap, finalTableList +} + +func finalizeTableColumnList(finalTableList []sqlname.NameTuple) ([]sqlname.NameTuple, *utils.StructMap[sqlname.NameTuple, []string]) { if changeStreamingIsEnabled(exportType) { reportUnsupportedTables(finalTableList) } - log.Infof("initial all tables table list for data export: %v", tableList) + log.Infof("initial all tables table list for data export: %v", finalTableList) + var skippedTableList []sqlname.NameTuple if !changeStreamingIsEnabled(exportType) { finalTableList, skippedTableList = source.DB().FilterEmptyTables(finalTableList) if len(skippedTableList) != 0 { @@ -787,13 +825,6 @@ func getFinalTableColumnList() (map[string]string, []sqlname.NameTuple, *utils.S })) } - var partitionsToRootTableMap map[string]string - isTableListSet := source.TableList != "" - partitionsToRootTableMap, finalTableList, err = addLeafPartitionsInTableList(finalTableList, isTableListSet) - if err != nil { - utils.ErrExit("failed to add the leaf partitions in table list: %w", err) - } - tablesColumnList, unsupportedTableColumnsMap, err := source.DB().GetColumnsWithSupportedTypes(finalTableList, useDebezium, changeStreamingIsEnabled(exportType)) if err != nil { utils.ErrExit("get columns with supported types: %v", err) @@ -823,7 +854,7 @@ func getFinalTableColumnList() (map[string]string, []sqlname.NameTuple, *utils.S finalTableList = filterTableWithEmptySupportedColumnList(finalTableList, tablesColumnList) } - return partitionsToRootTableMap, finalTableList, tablesColumnList + return finalTableList, tablesColumnList } func exportDataOffline(ctx context.Context, cancel context.CancelFunc, finalTableList []sqlname.NameTuple, tablesColumnList *utils.StructMap[sqlname.NameTuple, []string], snapshotName string) error { @@ -1027,7 +1058,6 @@ func extractTableListFromString(fullTableList []sqlname.NameTuple, flagTableList return result } tableList := utils.CsvStringToSlice(flagTableList) - var unqualifiedTables []string var unknownTableNames []string for _, pattern := range tableList { tables := findPatternMatchingTables(pattern) @@ -1036,9 +1066,6 @@ func extractTableListFromString(fullTableList []sqlname.NameTuple, flagTableList } result = append(result, tables...) } - if len(unqualifiedTables) > 0 { - utils.ErrExit("Qualify following table names %v in the %s list with schema name", unqualifiedTables, listName) - } if len(unknownTableNames) > 0 { utils.PrintAndLog("Unknown table names %v in the %s list", unknownTableNames, listName) utils.ErrExit("Valid table names are %v", lo.Map(fullTableList, func(tableName sqlname.NameTuple, _ int) string { diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index c594f7b997..b698654329 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -134,6 +134,7 @@ func exportSchema() error { // Check if the source database has the required permissions for exporting schema. if source.RunGuardrailsChecks { + checkIfSchemasHaveUsagePermissions() missingPerms, err := source.DB().GetMissingExportSchemaPermissions() if err != nil { return fmt.Errorf("failed to get missing migration permissions: %w", err) diff --git a/yb-voyager/src/namereg/namereg.go b/yb-voyager/src/namereg/namereg.go index 46f35ce464..12ff9045cb 100644 --- a/yb-voyager/src/namereg/namereg.go +++ b/yb-voyager/src/namereg/namereg.go @@ -158,7 +158,7 @@ func (reg *NameRegistry) registerSourceNames() (bool, error) { m[schemaName] = tableNames seqNames, err := reg.params.SDB.GetAllSequencesRaw(schemaName) if err != nil { - return false, fmt.Errorf("get all table names: %w", err) + return false, fmt.Errorf("get all sequence names: %w", err) } m[schemaName] = append(m[schemaName], seqNames...) } @@ -211,7 +211,7 @@ func (reg *NameRegistry) registerYBNames() (bool, error) { m[schemaName] = tableNames seqNames, err := yb.GetAllSequencesRaw(schemaName) if err != nil { - return false, fmt.Errorf("get all table names: %w", err) + return false, fmt.Errorf("get all sequence names: %w", err) } m[schemaName] = append(m[schemaName], seqNames...) } diff --git a/yb-voyager/src/srcdb/mysql.go b/yb-voyager/src/srcdb/mysql.go index 216821045e..7bad4333b6 100644 --- a/yb-voyager/src/srcdb/mysql.go +++ b/yb-voyager/src/srcdb/mysql.go @@ -549,3 +549,7 @@ func (ms *MySQL) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCo func (ms *MySQL) GetMissingAssessMigrationPermissions() ([]string, error) { return nil, nil } + +func (ms *MySQL) GetSchemasMissingUsagePermissions() ([]string, error) { + return nil, nil +} diff --git a/yb-voyager/src/srcdb/oracle.go b/yb-voyager/src/srcdb/oracle.go index 2af042e9cf..47412fa709 100644 --- a/yb-voyager/src/srcdb/oracle.go +++ b/yb-voyager/src/srcdb/oracle.go @@ -732,3 +732,7 @@ func (ora *Oracle) GetMissingAssessMigrationPermissions() ([]string, error) { func (ora *Oracle) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) { return false, 0, 0, nil } + +func (ora *Oracle) GetSchemasMissingUsagePermissions() ([]string, error) { + return nil, nil +} diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 21d4cd2813..b619d1f722 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -654,6 +654,7 @@ func (pg *PostgreSQL) GetColumnsWithSupportedTypes(tableList []sqlname.NameTuple func (pg *PostgreSQL) ParentTableOfPartition(table sqlname.NameTuple) string { var parentTable string + // For this query in case of case sensitive tables, minquoting is required query := fmt.Sprintf(`SELECT inhparent::pg_catalog.regclass FROM pg_catalog.pg_class c JOIN pg_catalog.pg_inherits ON c.oid = inhrelid @@ -1018,15 +1019,6 @@ Returns: func (pg *PostgreSQL) GetMissingExportSchemaPermissions() ([]string, error) { var combinedResult []string - // Check if schemas have USAGE permission - missingSchemas, err := pg.listSchemasMissingUsagePermission() - if err != nil { - return nil, fmt.Errorf("error checking schema usage permissions: %w", err) - } - if len(missingSchemas) > 0 { - combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString("Missing USAGE permission for user %s on Schemas: ", pg.source.User), strings.Join(missingSchemas, ", "))) - } - // Check if tables have SELECT permission missingTables, err := pg.listTablesMissingSelectPermission() if err != nil { @@ -1101,15 +1093,6 @@ func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string) ([]stri combinedResult = append(combinedResult, fmt.Sprintf("\n%sCREATE on database %s", color.RedString("Missing permission for user "+pg.source.User+": "), pg.source.DBName)) } - // Check if schemas have USAGE permission - missingSchemas, err := pg.listSchemasMissingUsagePermission() - if err != nil { - return nil, fmt.Errorf("error checking schema usage permissions: %w", err) - } - if len(missingSchemas) > 0 { - combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString(fmt.Sprintf("Missing USAGE permission for user %s on Schemas: ", pg.source.User)), strings.Join(missingSchemas, ", "))) - } - // Check replica identity of tables // missingTables, err := pg.listTablesMissingReplicaIdentityFull() // if err != nil { @@ -1161,15 +1144,6 @@ func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string) ([]stri func (pg *PostgreSQL) GetMissingAssessMigrationPermissions() ([]string, error) { var combinedResult []string - // Check if schemas have USAGE permission - missingSchemas, err := pg.listSchemasMissingUsagePermission() - if err != nil { - return nil, fmt.Errorf("error checking schema usage permissions: %w", err) - } - if len(missingSchemas) > 0 { - combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString("Missing USAGE permission for user %s on Schemas: ", pg.source.User), strings.Join(missingSchemas, ", "))) - } - // Check if tables have SELECT permission missingTables, err := pg.listTablesMissingSelectPermission() if err != nil { @@ -1536,7 +1510,7 @@ func (pg *PostgreSQL) listTablesMissingSelectPermission() (tablesWithMissingPerm return tablesWithMissingPerm, nil } -func (pg *PostgreSQL) listSchemasMissingUsagePermission() ([]string, error) { +func (pg *PostgreSQL) GetSchemasMissingUsagePermissions() ([]string, error) { // Users need usage permissions on the schemas they want to export and the pg_catalog and information_schema schemas trimmedSchemaList := pg.getTrimmedSchemaList() trimmedSchemaList = append(trimmedSchemaList, "pg_catalog", "information_schema") diff --git a/yb-voyager/src/srcdb/srcdb.go b/yb-voyager/src/srcdb/srcdb.go index 6ff1934e7d..d16ac0d104 100644 --- a/yb-voyager/src/srcdb/srcdb.go +++ b/yb-voyager/src/srcdb/srcdb.go @@ -61,6 +61,7 @@ type SourceDB interface { GetMissingExportDataPermissions(exportType string) ([]string, error) GetMissingAssessMigrationPermissions() ([]string, error) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) + GetSchemasMissingUsagePermissions() ([]string, error) } func newSourceDB(source *Source) SourceDB { diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index fc2a6c7a71..1bc9256037 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -1048,3 +1048,7 @@ func (yb *YugabyteDB) GetMissingAssessMigrationPermissions() ([]string, error) { func (yb *YugabyteDB) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) { return checkReplicationSlotsForPGAndYB(yb.db) } + +func (yb *YugabyteDB) GetSchemasMissingUsagePermissions() ([]string, error) { + return nil, nil +} From 284a2802dd9d97c918ba5b34d1b3cf22d6d714bf Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:29:36 +0530 Subject: [PATCH 009/105] Added check for java as a dependency in guardrails (#1919) --- yb-voyager/cmd/export.go | 85 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/yb-voyager/cmd/export.go b/yb-voyager/cmd/export.go index f5a43dc67f..4f03cfe842 100644 --- a/yb-voyager/cmd/export.go +++ b/yb-voyager/cmd/export.go @@ -18,6 +18,8 @@ package cmd import ( "fmt" "os" + "os/exec" + "strconv" "strings" "github.com/spf13/cobra" @@ -32,6 +34,8 @@ import ( // source struct will be populated by CLI arguments parsing var source srcdb.Source +const MIN_REQUIRED_JAVA_VERSION = 17 + // to disable progress bar during data export and import var disablePb utils.BoolStr var exportType string @@ -395,21 +399,35 @@ func checkDependenciesForExport() (binaryCheckIssues []string, err error) { if err != nil { return nil, err } else if binaryCheckIssue != "" { - binaryCheckIssues = append(binaryCheckIssues, binaryCheckIssue) } } - if len(binaryCheckIssues) > 0 { - binaryCheckIssues = append(binaryCheckIssues, "Install or Add the required dependencies to PATH and try again\n") + } + + if changeStreamingIsEnabled(exportType) || useDebezium { + // Check for java + javaIssue, err := checkJavaVersion() + if err != nil { + return nil, err + } + if javaIssue != "" { + binaryCheckIssues = append(binaryCheckIssues, javaIssue) } } + if len(binaryCheckIssues) > 0 { + binaryCheckIssues = append(binaryCheckIssues, "Install or Add the required dependencies to PATH and try again") + } + if changeStreamingIsEnabled(exportType) || useDebezium { // Check for debezium // FindDebeziumDistribution returns an error only if the debezium distribution is not found // So its error mesage will be added to problems - err := dbzm.FindDebeziumDistribution(source.DBType, false) + err = dbzm.FindDebeziumDistribution(source.DBType, false) if err != nil { + if len(binaryCheckIssues) > 0 { + binaryCheckIssues = append(binaryCheckIssues, "") + } binaryCheckIssues = append(binaryCheckIssues, strings.ToUpper(err.Error()[:1])+err.Error()[1:]) binaryCheckIssues = append(binaryCheckIssues, "Please check your Voyager installation and try again") } @@ -417,3 +435,62 @@ func checkDependenciesForExport() (binaryCheckIssues []string, err error) { return binaryCheckIssues, nil } + +func checkJavaVersion() (binaryCheckIssue string, err error) { + javaBinary := "java" + if javaHome := os.Getenv("JAVA_HOME"); javaHome != "" { + javaBinary = javaHome + "/bin/java" + } + + // Execute `java -version` to get the version + cmd := exec.Command(javaBinary, "-version") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Sprintf("java: required version >= %d", MIN_REQUIRED_JAVA_VERSION), nil + } + + // Example output + // java version "11.0.16" 2022-07-19 LTS + // Java(TM) SE Runtime Environment (build 11.0.16+8-LTS-211) + // Java HotSpot(TM) 64-Bit Server VM (build 11.0.16+8-LTS-211, mixed mode, sharing) + + // Convert output to string + versionOutput := string(output) + + // Extract the line with the version + var versionLine string + lines := strings.Split(versionOutput, "\n") + for _, line := range lines { + if strings.Contains(line, "version") { + versionLine = line + break + } + } + if versionLine == "" { + return "", fmt.Errorf("unable to find java version in output: %s", versionOutput) + } + + // Extract version string from the line (mimics awk -F '"' '/version/ {print $2}') + startIndex := strings.Index(versionLine, "\"") + endIndex := strings.LastIndex(versionLine, "\"") + if startIndex == -1 || endIndex == -1 || startIndex >= endIndex { + return "", fmt.Errorf("unexpected java version output: %s", versionOutput) + } + version := versionLine[startIndex+1 : endIndex] + + // Extract major version + versionNumbers := strings.Split(version, ".") + if len(versionNumbers) < 1 { + return "", fmt.Errorf("unexpected java version output: %s", versionOutput) + } + majorVersion, err := strconv.Atoi(versionNumbers[0]) + if err != nil { + return "", fmt.Errorf("unexpected java version output: %s", versionOutput) + } + + if majorVersion < MIN_REQUIRED_JAVA_VERSION { + return fmt.Sprintf("Java version %s is not supported. Please install Java version %d or higher", version, MIN_REQUIRED_JAVA_VERSION), nil + } + + return "", nil +} From 443414cba4caeeb8ec47f9130d6b08239703a12a Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:30:33 +0530 Subject: [PATCH 010/105] Introduced a flag --version into the installer script to install the specified released version of voyager (#1908) - This change is required to test rc releases since they won't be marked as latest from now on on GH. - Moreover, it gives the developers and customers the freedom to install the version of their choice using installer script. - The flag works like: --version 1.8.5 or -V 1.8.5 --- installer_scripts/install-yb-voyager | 103 ++++++++++++++++----------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/installer_scripts/install-yb-voyager b/installer_scripts/install-yb-voyager index 86185127c7..f026999f59 100755 --- a/installer_scripts/install-yb-voyager +++ b/installer_scripts/install-yb-voyager @@ -131,10 +131,10 @@ centos_main() { fi # TODO: Remove the usage of jq and use something inbuilt in the future. - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then $YUM_INSTALL jq 1>&2 - fetch_latest_release_data + fetch_release_data fi centos_check_base_repo_enabled @@ -177,10 +177,10 @@ ubuntu_main() { sudo apt-get update 1>&2 # TODO: Remove the usage of jq and use something inbuilt in the future. - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then sudo apt-get install -y jq 1>&2 - fetch_latest_release_data + fetch_release_data fi output "Installing packages." @@ -225,10 +225,10 @@ macos_main() { macos_install_brew # TODO: Remove the usage of jq and use something inbuilt in the future. - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then brew install jq 1>&2 - fetch_latest_release_data + fetch_release_data fi macos_install_pg_dump @@ -248,41 +248,56 @@ macos_main() { # COMMON #============================================================================= -# Function to fetch the latest release data from GitHub API -fetch_latest_release_data() { - # Fetch the latest release data from GitHub API - LATEST_RELEASE_DATA=$(curl -s https://api.github.com/repos/yugabyte/yb-voyager/releases/latest) - if [ -z "$LATEST_RELEASE_DATA" ]; then - echo "ERROR: Failed to fetch the latest release data from the GitHub API." - exit 1 - fi +# Function to fetch the release data from GitHub API +fetch_release_data() { + # Fetch the latest release data from GitHub API if VERSION is latest + if [ "${VERSION}" == "latest" ]; then + RELEASE_DATA=$(curl -s https://api.github.com/repos/yugabyte/yb-voyager/releases/latest) + if [ -z "$RELEASE_DATA" ]; then + echo "ERROR: Failed to fetch the latest release data from the GitHub API." + exit 1 + fi + fi - # Extract the latest release name and tag name - LATEST_RELEASE_NAME=$(echo "$LATEST_RELEASE_DATA" | jq -r '.name') - LATEST_TAG_NAME=$(echo "$LATEST_RELEASE_DATA" | jq -r '.tag_name') + # Extract the latest release name and tag name from the fetched data if VERSION is latest + if [ "${VERSION}" == "latest" ]; then + RELEASE_NAME=$(echo "$RELEASE_DATA" | jq -r '.name') + TAG_NAME=$(echo "$RELEASE_DATA" | jq -r '.tag_name') + else + # If the version is not latest, then the provided VERSION is used in the release name and tag name + RELEASE_NAME="v${VERSION}" + TAG_NAME="yb-voyager/v${VERSION}" + fi - # Fetch the commit hash of the latest tagged commit - LATEST_TAG_DATA=$(curl -s https://api.github.com/repos/yugabyte/yb-voyager/git/refs/tags/${LATEST_TAG_NAME}) - if [ -z "$LATEST_TAG_DATA" ]; then - echo "ERROR: Failed to fetch the latest tagged commit data from the GitHub API." + # Fetch the commit hash of the tagged commit related to the release + TAG_DATA=$(curl -s https://api.github.com/repos/yugabyte/yb-voyager/git/refs/tags/${TAG_NAME}) + if [ -z "$TAG_DATA" ]; then + echo "ERROR: Failed to fetch the tagged commit data from the GitHub API." exit 1 fi - LATEST_TAGGED_COMMIT=$(echo "$LATEST_TAG_DATA" | jq -r '.object.sha') + TAGGED_COMMIT=$(echo "$TAG_DATA" | jq -r '.object.sha') # Extract voyager version from the release name - VOYAGER_VERSION=$(echo "$LATEST_RELEASE_NAME" | sed 's/v//') + VOYAGER_RELEASE_VERSION=$(echo "$RELEASE_NAME" | sed 's/v//') # Log the fetched data to the log file - echo "LATEST_RELEASE_NAME=${LATEST_RELEASE_NAME}" >> "$LOG_FILE" - echo "LATEST_TAG_NAME=${LATEST_TAG_NAME}" >> "$LOG_FILE" - echo "LATEST_TAGGED_COMMIT=${LATEST_TAGGED_COMMIT}" >> "$LOG_FILE" - echo "VOYAGER_VERSION=${VOYAGER_VERSION}" >> "$LOG_FILE" + echo "RELEASE_NAME=${RELEASE_NAME}" >> "$LOG_FILE" + echo "TAG_NAME=${TAG_NAME}" >> "$LOG_FILE" + echo "TAGGED_COMMIT=${TAGGED_COMMIT}" >> "$LOG_FILE" + echo "VOYAGER_RELEASE_VERSION=${VOYAGER_RELEASE_VERSION}" >> "$LOG_FILE" # Set global variables for version and hash - VOYAGER_RELEASE_NAME=${VOYAGER_RELEASE_NAME:-${LATEST_RELEASE_NAME}} - DEBEZIUM_VERSION=${DEBEZIUM_VERSION:-"2.5.2-${VOYAGER_VERSION}"} - YB_VOYAGER_GIT_HASH=${LATEST_TAGGED_COMMIT} + VOYAGER_RELEASE_NAME=${RELEASE_NAME} + DEBEZIUM_VERSION="${DEBEZIUM_LOCAL_VERSION}-${VOYAGER_RELEASE_VERSION}" + # If voyager version contains 0rcx then DEBEZIUM_VERSION will be like 0rcx.2.5.2-[voyager version without rcx] + if [[ $VOYAGER_RELEASE_VERSION == *"rc"* ]]; then + # In case rc release voyager version is like 0rc1.1.8.5 + RC_PREFIX="${VOYAGER_RELEASE_VERSION:0:4}" + VOYAGER_VERSION="${VOYAGER_RELEASE_VERSION:5}" + DEBEZIUM_VERSION="${RC_PREFIX}.${DEBEZIUM_LOCAL_VERSION}-${VOYAGER_VERSION}" + fi + YB_VOYAGER_GIT_HASH=${TAGGED_COMMIT} } check_java() { @@ -308,10 +323,10 @@ check_java() { } install_debezium_server(){ - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then - output "Installing debezium:${VERSION}:${DEBEZIUM_VERSION}" - install_debezium_server_latest_release + output "Installing debezium:${DEBEZIUM_VERSION}" + install_debezium_server_from_release return fi output "Installing debezium:${VERSION}." @@ -330,7 +345,7 @@ install_debezium_server(){ package_debezium_server_local } -install_debezium_server_latest_release() { +install_debezium_server_from_release() { debezium_server_filename="debezium-server.tar.gz" # download wget -nv "https://github.com/yugabyte/yb-voyager/releases/download/yb-voyager/${VOYAGER_RELEASE_NAME}/${debezium_server_filename}" @@ -523,9 +538,9 @@ rebuild_voyager_local() { get_passed_options() { if [ "$1" == "linux" ] then - OPTS=$(getopt -o "lpv", --long install-from-local-source,only-pg-support,rebuild-voyager-local --name 'install-yb-voyager' -- $ARGS_LINUX) + OPTS=$(getopt -o "lpvV", --long install-from-local-source,only-pg-support,rebuild-voyager-local,version: --name 'install-yb-voyager' -- $ARGS_LINUX) else - OPTS=$(getopt lpv $ARGS_MACOS) + OPTS=$(getopt lpvV $ARGS_MACOS) fi eval set -- "$OPTS" @@ -544,6 +559,10 @@ get_passed_options() { REBUILD_VOYAGER_LOCAL="true"; shift ;; + -V | --version ) + VERSION="$2" + shift 2 + ;; * ) break ;; @@ -629,9 +648,9 @@ update_yb_voyager_bashrc() { install_yb_voyager() { GO=${GO:-"go"} - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then - output "Installing yb-voyager:${VERSION}:${VOYAGER_VERSION}" + output "Installing yb-voyager:${VOYAGER_RELEASE_VERSION}" $GO install github.com/yugabyte/yb-voyager/yb-voyager@${YB_VOYAGER_GIT_HASH} sudo mv -f $HOME/go/bin/yb-voyager /usr/local/bin return @@ -801,7 +820,7 @@ create_base_ora2pg_conf_file() { fi output "Installing the latest base-ora2pg.conf" - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then sudo wget -nv -O $conf_file_name https://github.com/yugabyte/yb-voyager/raw/$YB_VOYAGER_GIT_HASH/yb-voyager/src/srcdb/data/sample-ora2pg.conf else @@ -823,7 +842,7 @@ create_pg_dump_args_file() { fi output "Installing the latest pg_dump-args.ini" - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then sudo wget -nv -O $args_file_name https://github.com/yugabyte/yb-voyager/raw/$YB_VOYAGER_GIT_HASH/yb-voyager/src/srcdb/data/pg_dump-args.ini else @@ -840,7 +859,7 @@ create_gather_assessment_metadata_dir() { sudo mkdir -p $scripts_parent_dir output "Installing the latest scripts for gathering assessment metadata" - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then TAR_URL="https://github.com/yugabyte/yb-voyager/raw/$YB_VOYAGER_GIT_HASH/$scripts_dir_path/${scripts_dir_name}.tar.gz" sudo wget -nv -O /tmp/${scripts_dir_name}.tar.gz $TAR_URL @@ -860,7 +879,7 @@ create_guardrail_scripts_dir() { sudo mkdir -p $scripts_parent_dir/$scripts_dir_name output "Installing the guardrails scripts" - if [ "${VERSION}" == "latest" ] + if [ "${VERSION}" != "local" ] then TAR_URL="https://github.com/yugabyte/yb-voyager/archive/$YB_VOYAGER_GIT_HASH.tar.gz" sudo wget -nv -O /tmp/yb-voyager.tar.gz $TAR_URL From 976f42c5b9c2f31a05918cca72075d4e40581040 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 21 Nov 2024 13:35:42 +0530 Subject: [PATCH 011/105] Reporting PK and Unique constraints on complex datatypes in analyze and assessment (#1921) 1. Reporting Indexes on a few more datatypes like DATERANGE, TSRANGE, TSTZRANGE, NUMRANGE,INT4RANGE, INT8RANGE, INTERVAL (YEAR TO MONTH and DAY TO SECOND). 2. Reporting Unique and PRIMARY KEY constraints on all the unsupported index datatypes. https://yugabyte.atlassian.net/browse/DB-13973 --- .../schema/tables/INDEXES_table.sql | 15 ++ .../dummy-export-dir/schema/tables/table.sql | 24 +- .../tests/analyze-schema/expected_issues.json | 227 +++++++++++++++--- migtests/tests/analyze-schema/summary.json | 10 +- .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 92 ++++++- .../pg_assessment_report.sql | 14 +- .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 4 + .../expected_schema_analysis_report.json | 2 +- .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 4 + .../expectedAssessmentReport.json | 4 + yb-voyager/cmd/analyzeSchema.go | 130 +++++++++- yb-voyager/cmd/assessMigrationCommand.go | 4 + yb-voyager/cmd/constants.go | 2 + 19 files changed, 493 insertions(+), 63 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql index f3496e6089..efc811b0f2 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql @@ -81,6 +81,21 @@ CREATE INDEX idx14 on combined_tbl (bitt); CREATE INDEX idx15 on combined_tbl (bittv); +CREATE INDEX idx1 on combined_tbl1 (d); + +CREATE INDEX idx2 on combined_tbl1 (t); + +CREATE INDEX idx3 on combined_tbl1 (tz); + +CREATE INDEX idx4 on combined_tbl1 (n); + +CREATE INDEX idx5 on combined_tbl1 (i4); + +CREATE INDEX idx6 on combined_tbl1 (i8); + +CREATE INDEX idx7 on combined_tbl1 (inym); + +CREATE INDEX idx8 on combined_tbl1 (inds); CREATE INDEX idx_udt on test_udt(home_address); diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index e677a36f2d..3af667c481 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -321,7 +321,7 @@ create table combined_tbl ( c cidr, ci circle, b box, - j json, + j json UNIQUE, l line, ls lseg, maddr macaddr, @@ -332,9 +332,29 @@ create table combined_tbl ( p2 polygon, id1 txid_snapshot, bitt bit (13), - bittv bit varying(15) + bittv bit varying(15), + CONSTRAINT pk PRIMARY KEY (id, maddr8) ); +ALTER TABLE combined_tbl + ADD CONSTRAINT combined_tbl_unique UNIQUE(id, bitt); + +CREATE TABLE combined_tbl1( + id int, + t tsrange, + d daterange, + tz tstzrange, + n numrange, + i4 int4range UNIQUE, + i8 int8range, + inym INTERVAL YEAR TO MONTH, + inds INTERVAL DAY TO SECOND(9), + PRIMARY KEY(id, t, n) +); + +ALTER TABLE combined_tbl1 + ADD CONSTRAINT combined_tbl1_unique UNIQUE(id, d); + CREATE UNLOGGED TABLE tbl_unlogged (id int, val text); CREATE TABLE test_udt ( diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index b88b235326..0c1dc18255 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -25,7 +25,7 @@ "Reason": "INDEX on column 'cidr' not yet supported", "SqlStatement": "CREATE index idx1 on combined_tbl (c);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -35,7 +35,7 @@ "Reason": "INDEX on column 'circle' not yet supported", "SqlStatement": "CREATE index idx2 on combined_tbl (ci);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -45,7 +45,7 @@ "Reason": "INDEX on column 'box' not yet supported", "SqlStatement": "CREATE index idx3 on combined_tbl (b);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -55,7 +55,7 @@ "Reason": "INDEX on column 'json' not yet supported", "SqlStatement": "CREATE index idx4 on combined_tbl (j);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -65,7 +65,7 @@ "Reason": "INDEX on column 'line' not yet supported", "SqlStatement": "CREATE index idx5 on combined_tbl (l);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -75,7 +75,7 @@ "Reason": "INDEX on column 'lseg' not yet supported", "SqlStatement": "CREATE index idx6 on combined_tbl (ls);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -85,7 +85,7 @@ "Reason": "INDEX on column 'macaddr' not yet supported", "SqlStatement": "CREATE index idx7 on combined_tbl (maddr);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -95,7 +95,7 @@ "Reason": "INDEX on column 'macaddr8' not yet supported", "SqlStatement": "CREATE index idx8 on combined_tbl (maddr8);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -105,7 +105,7 @@ "Reason": "INDEX on column 'point' not yet supported", "SqlStatement": "CREATE index idx9 on combined_tbl (p);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -115,7 +115,7 @@ "Reason": "INDEX on column 'pg_lsn' not yet supported", "SqlStatement": "CREATE index idx10 on combined_tbl (lsn);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -125,7 +125,7 @@ "Reason": "INDEX on column 'path' not yet supported", "SqlStatement": "CREATE index idx11 on combined_tbl (p1);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -135,7 +135,7 @@ "Reason": "INDEX on column 'polygon' not yet supported", "SqlStatement": "CREATE index idx12 on combined_tbl (p2);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -145,7 +145,7 @@ "Reason": "INDEX on column 'txid_snapshot' not yet supported", "SqlStatement": "CREATE index idx13 on combined_tbl (id1);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -155,7 +155,7 @@ "Reason": "INDEX on column 'bit' not yet supported", "SqlStatement": "CREATE INDEX idx14 on combined_tbl (bitt);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -165,7 +165,7 @@ "Reason": "INDEX on column 'varbit' not yet supported", "SqlStatement": "CREATE INDEX idx15 on combined_tbl (bittv);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -235,7 +235,7 @@ "Reason": "INDEX on column 'user_defined_type' not yet supported", "SqlStatement": "CREATE INDEX idx_udt on test_udt(home_address);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -245,7 +245,7 @@ "Reason": "INDEX on column 'user_defined_type' not yet supported", "SqlStatement": "CREATE INDEX idx_udt1 on test_udt(home_address1);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -255,7 +255,7 @@ "Reason": "INDEX on column 'user_defined_type' not yet supported", "SqlStatement": "CREATE INDEX \"idx\u0026_enum2\" on test_udt((some_field::non_public.enum_test));", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -645,14 +645,165 @@ "Suggestion": "", "GH": "https://github.com/yugabyte/yugabyte-db/issues/10273" }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx1 ON combined_tbl1", + "Reason": "INDEX on column 'daterange' not yet supported", + "SqlStatement": "CREATE INDEX idx1 on combined_tbl1 (d);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx2 ON combined_tbl1", + "Reason": "INDEX on column 'tsrange' not yet supported", + "SqlStatement": "CREATE INDEX idx2 on combined_tbl1 (t);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx3 ON combined_tbl1", + "Reason": "INDEX on column 'tstzrange' not yet supported", + "SqlStatement": "CREATE INDEX idx3 on combined_tbl1 (tz);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx4 ON combined_tbl1", + "Reason": "INDEX on column 'numrange' not yet supported", + "SqlStatement": "CREATE INDEX idx4 on combined_tbl1 (n);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx5 ON combined_tbl1", + "Reason": "INDEX on column 'int4range' not yet supported", + "SqlStatement": "CREATE INDEX idx5 on combined_tbl1 (i4);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx6 ON combined_tbl1", + "Reason": "INDEX on column 'int8range' not yet supported", + "SqlStatement": "CREATE INDEX idx6 on combined_tbl1 (i8);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "test_interval", - "Reason": "PRIMARY KEY containing column of type 'INTERVAL' not yet supported.", + "ObjectName": "combined_tbl, constraint: combined_tbl_j_key", + "Reason": "Primary key and Unique constraint on column 'json' not yet supported", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "combined_tbl, constraint: pk", + "Reason": "Primary key and Unique constraint on column 'macaddr8' not yet supported", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "combined_tbl, constraint: combined_tbl_unique", + "Reason": "Primary key and Unique constraint on column 'bit' not yet supported", + "SqlStatement": "ALTER TABLE combined_tbl\n\t\tADD CONSTRAINT combined_tbl_unique UNIQUE(id, bitt);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "combined_tbl1, constraint: combined_tbl1_i4_key", + "Reason": "Primary key and Unique constraint on column 'int4range' not yet supported", + "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "combined_tbl1, constraint: combined_tbl1_id_t_n_pkey", + "Reason": "Primary key and Unique constraint on column 'tsrange' not yet supported", + "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "combined_tbl1, constraint: combined_tbl1_id_t_n_pkey", + "Reason": "Primary key and Unique constraint on column 'numrange' not yet supported", + "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "combined_tbl1, constraint: combined_tbl1_unique", + "Reason": "Primary key and Unique constraint on column 'daterange' not yet supported", + "SqlStatement": "ALTER TABLE combined_tbl1\n\t\tADD CONSTRAINT combined_tbl1_unique UNIQUE(id, d);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx7 ON combined_tbl1", + "Reason": "INDEX on column 'interval' not yet supported", + "SqlStatement": "CREATE INDEX idx7 on combined_tbl1 (inym);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "idx8 ON combined_tbl1", + "Reason": "INDEX on column 'interval' not yet supported", + "SqlStatement": "CREATE INDEX idx8 on combined_tbl1 (inds);", + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "test_interval, constraint: test_interval_frequency_pkey", + "Reason": "Primary key and Unique constraint on column 'interval' not yet supported", "SqlStatement": "create table test_interval(\n frequency interval primary key,\n\tcol1 int\n);", - "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1397" + "Suggestion": "Refer to the docs link for the workaround", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { "IssueType": "unsupported_features", @@ -737,7 +888,7 @@ "Reason": "INDEX on column 'tsvector' not yet supported", "SqlStatement": "CREATE INDEX tsvector_idx ON public.documents (title_tsvector, id);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -747,7 +898,7 @@ "Reason": "INDEX on column 'tsquery' not yet supported", "SqlStatement": "CREATE INDEX tsquery_idx ON public.ts_query_table (query);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -757,7 +908,7 @@ "Reason": "INDEX on column 'citext' not yet supported", "SqlStatement": "CREATE INDEX idx_citext ON public.citext_type USING btree (data);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -767,7 +918,7 @@ "Reason": "INDEX on column 'inet' not yet supported", "SqlStatement": "CREATE INDEX idx_inet ON public.inet_type USING btree (data);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -777,7 +928,7 @@ "Reason": "INDEX on column 'jsonb' not yet supported", "SqlStatement": "CREATE INDEX idx_json ON public.test_jsonb (data);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -787,7 +938,7 @@ "Reason": "INDEX on column 'jsonb' not yet supported", "SqlStatement": "CREATE INDEX idx_json2 ON public.test_jsonb ((data2::jsonb));", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -797,7 +948,7 @@ "Reason": "INDEX on column 'array' not yet supported", "SqlStatement": "create index idx_array on public.documents (list_of_sections);", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { @@ -901,7 +1052,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype - pg_lsn on column - lsn", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" @@ -911,7 +1062,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype - txid_snapshot on column - id1", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" @@ -951,7 +1102,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - circle on column - ci", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" @@ -961,7 +1112,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - box on column - b", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" @@ -971,7 +1122,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - line on column - l", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" @@ -981,7 +1132,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - lseg on column - ls", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" @@ -991,7 +1142,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - point on column - p", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" @@ -1001,7 +1152,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - path on column - p1", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" @@ -1011,7 +1162,7 @@ "ObjectType": "TABLE", "ObjectName": "combined_tbl", "Reason": "Unsupported datatype for Live migration - polygon on column - p2", - "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15)\n);", + "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 7486e054da..8b633466df 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -24,14 +24,14 @@ }, { "ObjectType": "TABLE", - "TotalCount": 48, + "TotalCount": 49, "InvalidCount": 33, - "ObjectNames": "test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, + "ObjectNames": "combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, { "ObjectType": "INDEX", - "TotalCount": 35, - "InvalidCount": 30, - "ObjectNames": "film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", + "TotalCount": 43, + "InvalidCount": 38, + "ObjectNames": "idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." }, { diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index fe91da122f..12884e9d65 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -655,6 +655,10 @@ "FeatureName": "View with check option", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 1d86531c65..665bb3f216 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -49,9 +49,9 @@ }, { "ObjectType": "INDEX", - "TotalCount": 24, - "InvalidCount": 20, - "ObjectNames": "idx1 ON public.combined_tbl, idx2 ON public.combined_tbl, idx3 ON public.combined_tbl, idx4 ON public.combined_tbl, idx5 ON public.combined_tbl, idx6 ON public.combined_tbl, idx7 ON public.combined_tbl, idx_array ON public.documents, idx_box_data ON public.mixed_data_types_table1, idx_box_data_brin ON public.mixed_data_types_table1, idx_citext ON public.citext_type, idx_citext1 ON public.citext_type, idx_citext2 ON public.citext_type, idx_inet ON public.inet_type, idx_inet1 ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_point_data ON public.mixed_data_types_table1, idx_valid ON public.test_jsonb, tsquery_idx ON public.ts_query_table, tsvector_idx ON public.documents, idx_box_data ON schema2.mixed_data_types_table1, idx_box_data_spgist ON schema2.mixed_data_types_table1, idx_point_data ON schema2.mixed_data_types_table1" + "TotalCount": 26, + "InvalidCount": 22, + "ObjectNames": "idx8 ON public.combined_tbl, idx9 ON public.combined_tbl, idx1 ON public.combined_tbl, idx2 ON public.combined_tbl, idx3 ON public.combined_tbl, idx4 ON public.combined_tbl, idx5 ON public.combined_tbl, idx6 ON public.combined_tbl, idx7 ON public.combined_tbl, idx_array ON public.documents, idx_box_data ON public.mixed_data_types_table1, idx_box_data_brin ON public.mixed_data_types_table1, idx_citext ON public.citext_type, idx_citext1 ON public.citext_type, idx_citext2 ON public.citext_type, idx_inet ON public.inet_type, idx_inet1 ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_point_data ON public.mixed_data_types_table1, idx_valid ON public.test_jsonb, tsquery_idx ON public.ts_query_table, tsvector_idx ON public.documents, idx_box_data ON schema2.mixed_data_types_table1, idx_box_data_spgist ON schema2.mixed_data_types_table1, idx_point_data ON schema2.mixed_data_types_table1" }, { "ObjectType": "FUNCTION", @@ -169,7 +169,7 @@ "test_views.abc_mview", "test_views.view_table1" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 70 objects (62 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 24 objects (5 tables/materialized views and 19 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 70 objects (62 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -473,6 +473,32 @@ { "ObjectName": "USER_DEFINED_TYPE: idx7 ON public.combined_tbl", "SqlStatement": "CREATE INDEX idx7 ON public.combined_tbl USING btree (address);" + }, + { + "ObjectName": "DATERANGE: idx8 ON public.combined_tbl", + "SqlStatement": "CREATE INDEX idx8 ON public.combined_tbl USING btree (d);" + }, + { + "ObjectName": "INTERVAL: idx9 ON public.combined_tbl", + "SqlStatement": "CREATE INDEX idx9 ON public.combined_tbl USING btree (inds3);" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [ + { + "ObjectName": "public.combined_tbl, constraint: combined_tbl_bittv_key", + "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT combined_tbl_bittv_key UNIQUE (bittv);" + }, + { + "ObjectName": "public.combined_tbl, constraint: uk", + "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT uk UNIQUE (lsn);" + }, + { + "ObjectName": "public.combined_tbl, constraint: combined_tbl_pkey", + "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT combined_tbl_pkey PRIMARY KEY (id, arr_enum);" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" @@ -624,7 +650,7 @@ "SchemaName": "public", "ObjectName": "combined_tbl", "RowCount": 0, - "ColumnCount": 9, + "ColumnCount": 11, "Reads": 0, "Writes": 0, "ReadsPerSecond": 0, @@ -1656,6 +1682,34 @@ "ParentTableName": "public.citext_type", "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "idx8", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.combined_tbl", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "idx9", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.combined_tbl", + "SizeInBytes": 8192 + }, { "SchemaName": "public", "ObjectName": "idx7", @@ -1698,6 +1752,34 @@ "ParentTableName": "public.combined_tbl", "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "combined_tbl_bittv_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.combined_tbl", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "uk", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.combined_tbl", + "SizeInBytes": 8192 + }, { "SchemaName": "public", "ObjectName": "idx4", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index fec2cb1edf..c826c33aaf 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -180,12 +180,18 @@ create table public.combined_tbl ( maddr macaddr, maddr8 macaddr8, lsn pg_lsn, + inds3 INTERVAL DAY TO SECOND(3), + d daterange, bitt bit (13), - bittv bit varying(15), + bittv bit varying(15) UNIQUE, address address_type, - arr_enum enum_kind[] + arr_enum enum_kind[], + PRIMARY KEY (id, arr_enum) ); +ALTER TABLE public.combined_tbl + ADD CONSTRAINT uk UNIQUE(lsn); + CREATE index idx1 on public.combined_tbl (c); CREATE index idx2 on public.combined_tbl (maddr); @@ -200,6 +206,10 @@ CREATE INDEX idx6 on public.combined_tbl (bittv); CREATE INDEX idx7 on public.combined_tbl (address); +CREATE INDEX idx8 on public.combined_tbl (d); + +CREATE INDEX idx9 on public.combined_tbl (inds3); + CREATE UNLOGGED TABLE tbl_unlogged (id int, val text); CREATE OR REPLACE FUNCTION public.check_sales_region() diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index d2151ea2f8..e977afb880 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -570,6 +570,10 @@ "FeatureName": "View with check option", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 17a32e2916..a6acd0153c 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -625,6 +625,10 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported" }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [ diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index 5e7ce60568..2c6e452ce7 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -226,7 +226,7 @@ "SqlStatement": "CREATE INDEX idx_1 ON composite_type_examples.ordinary_table USING btree (basic_);", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Refer to the docs link for the workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/9698", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" }, { diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 783a2ed7b7..83933b92ad 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -157,6 +157,10 @@ "FeatureName": "View with check option", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index ba92bad2b4..53c130bfd1 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -120,6 +120,10 @@ "FeatureName": "Gin indexes on multi-columns", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index dd13587bec..5b83660cca 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -891,6 +891,10 @@ "FeatureName": "View with check option", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index 092ca5739b..cb4ae18aee 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -222,6 +222,10 @@ "FeatureName": "View with check option", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index fbbfa6cadd..48942b5a41 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -128,6 +128,10 @@ "FeatureName": "Extensions", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 1c6f8dd993..c15829b277 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -332,6 +332,10 @@ "FeatureName": "View with check option", "Objects": [] }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [] + }, { "FeatureName": "Index on complex datatypes", "Objects": [] diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index a422e58a3c..618908d540 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -173,7 +173,6 @@ var ( likeRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, `\(LIKE`) inheritRegex = re("CREATE", opt(capture(unqualifiedIdent)), "TABLE", ifNotExists, capture(ident), anything, "INHERITS", "[ |(]") withOidsRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "WITH", anything, "OIDS") - intvlRegex = re("CREATE", "TABLE", ifNotExists, capture(ident)+`\(`, anything, "interval", "PRIMARY") anydataRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "AnyData", anything) anydatasetRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "AnyDataSet", anything) anyTypeRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "AnyType", anything) @@ -238,6 +237,7 @@ const ( POLICY_ROLE_ISSUE = "Policy require roles to be created." VIEW_CHECK_OPTION_ISSUE = "Schema containing VIEW WITH CHECK OPTION is not supported yet." ISSUE_INDEX_WITH_COMPLEX_DATATYPES = `INDEX on column '%s' not yet supported` + ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES = `Primary key and Unique constraint on column '%s' not yet supported` ISSUE_UNLOGGED_TABLE = "UNLOGGED tables are not supported yet." UNSUPPORTED_DATATYPE = "Unsupported datatype" UNSUPPORTED_DATATYPE_LIVE_MIGRATION = "Unsupported datatype for Live migration" @@ -392,9 +392,11 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { reportDeferrableConstraintCreateTable(createTableNode, sqlStmtInfo, fpath) reportUnsupportedDatatypes(createTableNode.CreateStmt.Relation, createTableNode.CreateStmt.TableElts, sqlStmtInfo, fpath, objType) parseColumnsWithUnsupportedIndexDatatypes(createTableNode) + reportUnsupportedConstraintsOnComplexDatatypesInCreate(createTableNode, sqlStmtInfo, fpath) reportUnloggedTable(createTableNode, sqlStmtInfo, fpath) } if isAlterTable { + reportUnsupportedConstraintsOnComplexDatatypesInAlter(alterTableNode, sqlStmtInfo, fpath) reportAlterAddPKOnPartition(alterTableNode, sqlStmtInfo, fpath) reportAlterTableVariants(alterTableNode, sqlStmtInfo, fpath, objType) reportExclusionConstraintAlterTable(alterTableNode, sqlStmtInfo, fpath) @@ -675,6 +677,13 @@ var UnsupportedIndexDatatypes = []string{ "cidr", "bit", // for BIT (n) "varbit", // for BIT varying (n) + "daterange", + "tsrange", + "tstzrange", + "numrange", + "int4range", + "int8range", + "interval", // same for INTERVAL YEAR TO MONTH and INTERVAL DAY TO SECOND //Below ones are not supported on PG as well with atleast btree access method. Better to have in our list though //Need to understand if there is other method or way available in PG to have these index key [TODO] "circle", @@ -689,6 +698,107 @@ var UnsupportedIndexDatatypes = []string{ // array as well but no need to add it in the list as fetching this type is a different way TODO: handle better with specific types } +func reportUnsupportedConstraintsOnComplexDatatypesInAlter(alterTableNode *pg_query.Node_AlterTableStmt, sqlStmtInfo sqlInfo, fpath string) { + schemaName := alterTableNode.AlterTableStmt.Relation.Schemaname + tableName := alterTableNode.AlterTableStmt.Relation.Relname + fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) + alterCmd := alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd() + unsupportedColumnsForTable, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] + if !ok { + return + } + if alterCmd.GetSubtype() != pg_query.AlterTableType_AT_AddConstraint { + return + } + if !slices.Contains([]pg_query.ConstrType{pg_query.ConstrType_CONSTR_PRIMARY, pg_query.ConstrType_CONSTR_UNIQUE}, + alterCmd.GetDef().GetConstraint().GetContype()) { + return + } + columns := alterCmd.GetDef().GetConstraint().GetKeys() + for _, col := range columns { + colName := col.GetString_().Sval + typeName, ok := unsupportedColumnsForTable[colName] + if ok { + displayName := fmt.Sprintf("%s, constraint: %s", fullyQualifiedName, alterCmd.GetDef().GetConstraint().GetConname()) + reportCase(fpath, fmt.Sprintf(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/25003", + "Refer to the docs link for the workaround", "TABLE", displayName, sqlStmtInfo.formattedStmt, + UNSUPPORTED_FEATURES, PK_UK_CONSTRAINT_ON_UNSUPPORTED_TYPE) + return + } + } +} + +func reportUnsupportedConstraintsOnComplexDatatypesInCreate(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { + schemaName := createTableNode.CreateStmt.Relation.Schemaname + tableName := createTableNode.CreateStmt.Relation.Relname + columns := createTableNode.CreateStmt.TableElts + fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) + unsupportedColumnsForTable, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] + if !ok { + return + } + reportConstraintIfrequired := func(con *pg_query.Constraint, colNames []string, typeName string) { + conType := con.GetContype() + if !slices.Contains([]pg_query.ConstrType{pg_query.ConstrType_CONSTR_PRIMARY, pg_query.ConstrType_CONSTR_UNIQUE}, conType) { + return + } + generatedConName := generateConstraintName(conType, fullyQualifiedName, colNames) + specifiedConstraintName := con.GetConname() + conName := lo.Ternary(specifiedConstraintName == "", generatedConName, specifiedConstraintName) + //report the PK / Unique constraint in CREATE TABLE on this column + summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true + reportCase(fpath, fmt.Sprintf(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/25003", + "Refer to the docs link for the workaround", "TABLE", fmt.Sprintf("%s, constraint: %s", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, + UNSUPPORTED_FEATURES, PK_UK_CONSTRAINT_ON_UNSUPPORTED_TYPE) + + } + for _, column := range columns { + if column.GetColumnDef() != nil { + /* + e.g. create table unique_def_test(id int, d daterange UNIQUE, c1 int); + create_stmt:{relation:{relname:"unique_def_test" inh:true relpersistence:"p" location:15}... + table_elts:{column_def:{colname:"d" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} + typemod:-1 location:34} is_local:true constraints:{constraint:{contype:CONSTR_UNIQUE location:38}} .... + + here checking the case where this clause is in column definition so iterating over each column_def and in that + constraint type is UNIQUE/ PK reporting that + supported. + */ + colName := column.GetColumnDef().GetColname() + typeName, ok := unsupportedColumnsForTable[colName] + if !ok { + continue + } + constraints := column.GetColumnDef().GetConstraints() + for _, c := range constraints { + reportConstraintIfrequired(c.GetConstraint(), []string{colName}, typeName) + } + } else if column.GetConstraint() != nil { + /* + e.g. create table uniquen_def_test1(id int, c1 citext, CONSTRAINT pk PRIMARY KEY(id, c)); + {create_stmt:{relation:{relname:"unique_def_test1" inh:true relpersistence:"p" location:80} table_elts:{column_def:{colname:"id" + type_name:{.... names:{string:{sval:"int4"}} typemod:-1 location:108} is_local:true location:105}} + table_elts:{constraint:{contype:CONSTR_UNIQUE deferrable:true initdeferred:true location:113 keys:{string:{sval:"id"}}}} .. + + here checking the case where this UK/ PK is at the end of column definition as a separate constraint + */ + keys := column.GetConstraint().GetKeys() + columns := []string{} + for _, k := range keys { + colName := k.GetString_().Sval + columns = append(columns, colName) + } + for _, c := range columns { + typeName, ok := unsupportedColumnsForTable[c] + if !ok { + continue + } + reportConstraintIfrequired(column.GetConstraint(), columns, typeName) + } + } + } +} + func parseColumnsWithUnsupportedIndexDatatypes(createTableNode *pg_query.Node_CreateStmt) { schemaName := createTableNode.CreateStmt.Relation.Schemaname tableName := createTableNode.CreateStmt.Relation.Relname @@ -738,14 +848,18 @@ func parseColumnsWithUnsupportedIndexDatatypes(createTableNode *pg_query.Node_Cr typeName, typeSchemaName := getTypeNameAndSchema(typeNames) fullTypeName := lo.Ternary(typeSchemaName != "", typeSchemaName+"."+typeName, typeName) colName := column.GetColumnDef().GetColname() - if len(column.GetColumnDef().GetTypeName().GetArrayBounds()) > 0 { + isArrayType := len(column.GetColumnDef().GetTypeName().GetArrayBounds()) > 0 + isUnsupportedType := slices.Contains(UnsupportedIndexDatatypes, typeName) + isUDTType := slices.Contains(compositeTypes, fullTypeName) + switch true { + case isArrayType: //For Array types and storing the type as "array" as of now we can enhance the to have specific type e.g. INT4ARRAY _, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] if !ok { columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] = make(map[string]string) } columnsWithUnsupportedIndexDatatypes[fullyQualifiedName][colName] = "array" - } else if slices.Contains(UnsupportedIndexDatatypes, typeName) || slices.Contains(compositeTypes, fullTypeName) { + case isUnsupportedType || isUDTType: _, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] if !ok { columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] = make(map[string]string) @@ -798,7 +912,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I typeName, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName][colName] if ok { summaryMap["INDEX"].invalidCount[displayObjName] = true - reportCase(fpath, fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/9698", + reportCase(fpath, fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/25003", "Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, INDEX_ON_UNSUPPORTED_TYPE) return @@ -810,7 +924,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I if len(param.GetIndexElem().GetExpr().GetTypeCast().GetTypeName().GetArrayBounds()) > 0 { //In case casting is happening for an array type summaryMap["INDEX"].invalidCount[displayObjName] = true - reportCase(fpath, fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "array"), "https://github.com/yugabyte/yugabyte-db/issues/9698", + reportCase(fpath, fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "array"), "https://github.com/yugabyte/yugabyte-db/issues/25003", "Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, INDEX_ON_UNSUPPORTED_TYPE) return @@ -820,7 +934,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I if slices.Contains(compositeTypes, fullCastTypeName) { reason = fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "user_defined_type") } - reportCase(fpath, reason, "https://github.com/yugabyte/yugabyte-db/issues/9698", + reportCase(fpath, reason, "https://github.com/yugabyte/yugabyte-db/issues/25003", "Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, INDEX_ON_UNSUPPORTED_TYPE) return @@ -1451,10 +1565,6 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "OIDs are not supported for user tables.", "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") - } else if tbl := intvlRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { - summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "PRIMARY KEY containing column of type 'INTERVAL' not yet supported.", - "https://github.com/YugaByte/yugabyte-db/issues/1397", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { reportCase(fpath, "ALTER TABLE OF not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 6930a1b29d..c4ca369c58 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -941,6 +941,10 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, UnsupportedIndexDatatypes)) + + pkOrUkConstraintIssuePrefix := strings.Split(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, "%s")[0] + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 262e4a0b35..5b19baaa82 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -157,6 +157,7 @@ const ( PARTITION_KEY_NOT_PK_DOC_LINK = DOCS_LINK_PREFIX + ORACLE_PREFIX + "#partition-key-column-not-part-of-primary-key-columns" DROP_TEMP_TABLE_DOC_LINK = DOCS_LINK_PREFIX + MYSQL_PREFIX + "#drop-temporary-table-statements-are-not-supported" INDEX_ON_UNSUPPORTED_TYPE = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#indexes-on-some-complex-data-types-are-not-supported" + PK_UK_CONSTRAINT_ON_UNSUPPORTED_TYPE = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#indexes-on-some-complex-data-types-are-not-supported" //Keeping it similar for now, will see if we need to a separate issue on docs UNLOGGED_TABLE_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unlogged-table-is-not-supported" XID_DATATYPE_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#xid-functions-is-not-supported" UNSUPPORTED_DATATYPES_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-yugabytedb" @@ -211,6 +212,7 @@ const ( UNLOGGED_TABLE_FEATURE = "Unlogged tables" REFERENCING_TRIGGER_FEATURE = "REFERENCING clause for triggers" BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables" + PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes" // Migration caveats From 1e12cf5c4ef36f88a2aebe703f060f6042200f5b Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 21 Nov 2024 15:26:01 +0530 Subject: [PATCH 012/105] Detect DML issues only from the PGSS queries for Unsupported Query Constructs in assess (#1954) https://yugabyte.atlassian.net/browse/DB-14071 --- .../tests/pg/assessment-report-test/init-db | 4 +-- yb-voyager/cmd/analyzeSchema.go | 2 +- yb-voyager/cmd/assessMigrationCommand.go | 2 +- yb-voyager/src/queryissue/queryissue.go | 19 +++++++----- yb-voyager/src/queryparser/query_parser.go | 30 ++++++++++++++----- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/init-db b/migtests/tests/pg/assessment-report-test/init-db index 07695dc83f..c4016a4c77 100755 --- a/migtests/tests/pg/assessment-report-test/init-db +++ b/migtests/tests/pg/assessment-report-test/init-db @@ -19,19 +19,17 @@ if [ ! -f "$TEMP_SQL" ]; then fi # Writing temporary SQL script to create DB objects for the test -# running the unsupported_quer_constructs.sql fileafter after all schema creation to nto get any DDL in pg_stat_statements cat < "$TEMP_SQL" \i ${TEST_DIR}/../misc-objects-1/schema.sql; \i ${TEST_DIR}/../misc-objects-2/pg_misc_objects2.sql; \i ${TEST_DIR}/../views-and-rules/pg_views_and_rules_automation.sql; \i pg_assessment_report.sql; +\i unsupported_query_constructs.sql; CREATE SCHEMA IF NOT EXISTS schema2; SET SEARCH_PATH TO schema2; \i ${TEST_DIR}/../misc-objects-1/schema.sql; \i ${TEST_DIR}/../misc-objects-2/pg_misc_objects2.sql; \i pg_assessment_report.sql; -SET SEARCH_PATH TO public; -\i unsupported_query_constructs.sql; EOF # Run the temporary SQL script diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 618908d540..50af2a43d6 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1700,7 +1700,7 @@ func checker(sqlInfoArr []sqlInfo, fpath string, objType string) { func checkPlPgSQLStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { for _, sqlInfoStmt := range sqlInfoArr { - issues, err := parserIssueDetector.GetIssues(sqlInfoStmt.formattedStmt) + issues, err := parserIssueDetector.GetAllIssues(sqlInfoStmt.formattedStmt) if err != nil { log.Infof("error in getting the issues-%s: %v", sqlInfoStmt.formattedStmt, err) continue diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index c4ca369c58..6bc2d2c8a8 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1103,7 +1103,7 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error query := executedQueries[i] log.Debugf("fetching unsupported query constructs for query - [%s]", query) - issues, err := parserIssueDetector.GetIssues(query) + issues, err := parserIssueDetector.GetDMLIssues(query) if err != nil { log.Errorf("failed while trying to fetch query issues in query - [%s]: %v", query, err) diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 9b9ca22a89..28b1b0ff4b 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -19,7 +19,6 @@ package queryissue import ( "fmt" - pg_query "github.com/pganalyze/pg_query_go/v5" "github.com/samber/lo" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" @@ -37,7 +36,7 @@ func NewParserIssueDetector() *ParserIssueDetector { return &ParserIssueDetector{} } -func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) GetAllIssues(query string) ([]issue.IssueInstance, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) @@ -50,7 +49,7 @@ func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, er } var issues []issue.IssueInstance for _, plpgsqlQuery := range plpgsqlQueries { - issuesInQuery, err := p.GetIssues(plpgsqlQuery) + issuesInQuery, err := p.GetAllIssues(plpgsqlQuery) if err != nil { //there can be plpgsql expr queries no parseable via parser e.g. "withdrawal > balance" log.Errorf("error getting issues in query-%s: %v", query, err) @@ -73,7 +72,7 @@ func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, er if err != nil { return nil, fmt.Errorf("error deparsing a select stmt: %v", err) } - issues, err := p.GetIssues(selectStmtQuery) + issues, err := p.GetAllIssues(selectStmtQuery) if err != nil { return nil, err } @@ -86,10 +85,16 @@ func (p *ParserIssueDetector) GetIssues(query string) ([]issue.IssueInstance, er }), nil } - return p.getDMLIssues(query, parseTree) + return p.GetDMLIssues(query) } -func (p *ParserIssueDetector) getDMLIssues(query string, parseTree *pg_query.ParseResult) ([]issue.IssueInstance, error) { +//TODO: in future when we will DDL issues detection here we need `GetDDLIssues` + +func (p *ParserIssueDetector) GetDMLIssues(query string) ([]issue.IssueInstance, error) { + parseTree, err := queryparser.Parse(query) + if err != nil { + return nil, fmt.Errorf("error parsing query: %w", err) + } var result []issue.IssueInstance var unsupportedConstructs []string visited := make(map[protoreflect.Message]bool) @@ -113,7 +118,7 @@ func (p *ParserIssueDetector) getDMLIssues(query string, parseTree *pg_query.Par } parseTreeProtoMsg := queryparser.GetProtoMessageFromParseTree(parseTree) - err := queryparser.TraverseParseTree(parseTreeProtoMsg, visited, processor) + err = queryparser.TraverseParseTree(parseTreeProtoMsg, visited, processor) if err != nil { return result, fmt.Errorf("error traversing parse tree message: %w", err) } diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/queryparser/query_parser.go index 8e0699d499..247a30c597 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/queryparser/query_parser.go @@ -65,9 +65,10 @@ func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect. return parseTree.Stmts[0].Stmt.ProtoReflect() } + func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE - _, isPlPgSQLObject := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) + _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) return isPlPgSQLObject } @@ -77,13 +78,13 @@ func IsViewObject(parseTree *pg_query.ParseResult) bool { } func IsMviewObject(parseTree *pg_query.ParseResult) bool { - createAsNode, isCreateAsStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) //for MVIEW case + createAsNode, isCreateAsStmt := getCreateTableAsStmtNode(parseTree) //for MVIEW case return isCreateAsStmt && createAsNode.CreateTableAsStmt.Objtype == pg_query.ObjectType_OBJECT_MATVIEW } func GetSelectStmtQueryFromViewOrMView(parseTree *pg_query.ParseResult) (string, error) { - viewNode, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) - createAsNode, _ := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt)//For MVIEW case + viewNode, isViewStmt := getCreateViewNode(parseTree) + createAsNode, _ := getCreateTableAsStmtNode(parseTree) //For MVIEW case var selectStmt *pg_query.SelectStmt if isViewStmt { selectStmt = viewNode.ViewStmt.GetQuery().GetSelectStmt() @@ -98,9 +99,9 @@ func GetSelectStmtQueryFromViewOrMView(parseTree *pg_query.ParseResult) (string, } func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { - createFuncNode, isCreateFunc := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) - viewNode, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) - createAsNode, _ := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) + createFuncNode, isCreateFunc := getCreateFuncStmtNode(parseTree) + viewNode, isViewStmt := getCreateViewNode(parseTree) + createAsNode, _ := getCreateTableAsStmtNode(parseTree) switch true { case isCreateFunc: /* @@ -148,3 +149,18 @@ func getFunctionObjectName(funcNameList []*pg_query.Node) string { } return lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) } + +func getCreateTableAsStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateTableAsStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) + return node, ok +} + +func getCreateViewNode(parseTree *pg_query.ParseResult) (*pg_query.Node_ViewStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) + return node, ok +} + +func getCreateFuncStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateFunctionStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) + return node, ok +} \ No newline at end of file From 7f1db9875d25483b70b7c5621cccdbf2dfc55d2f Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 21 Nov 2024 16:04:42 +0530 Subject: [PATCH 013/105] Implement RangeTableFunc detector for XMLTABLE identification (#1937) - Implement a new RangeTableFunc detector specifically targeting the detection of XMLTABLE() usage in SQL queries. - Note that while RangeTableFunc nodes can be used by various SQL constructs, this detector initially focuses on XMLTABLE(), with structured checks to validate this assumption. - The current implementation assumes XMLTABLE() as the primary use case for RangeTableFunc nodes but is designed to be adaptable for future extensions to other SQL constructs. Add unit tests and end-to-end tests for RangeTableFunc XMLTABLE detector chore: Update static analysis config to ignore S1008 - Add a configuration to staticcheck.conf to disable rule S1008. - This change preserves explicit return statements in functions for clearer logic understanding and maintenance. --- .../expectedAssessmentReport.json | 67 ++++- .../unsupported_query_constructs.sql | 113 +++++++- yb-voyager/src/queryissue/queryissue.go | 1 + .../queryissue/unsupported_dml_constructs.go | 29 ++ .../unsupported_dml_constructs_test.go | 250 +++++++++++++++--- yb-voyager/src/queryparser/helpers.go | 240 ++++++++++++++++- yb-voyager/src/queryparser/traversal.go | 18 +- yb-voyager/staticcheck.conf | 2 + 8 files changed, 656 insertions(+), 64 deletions(-) create mode 100644 yb-voyager/staticcheck.conf diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 665bb3f216..e3aa333e28 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -43,9 +43,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 64, - "InvalidCount": 20, - "ObjectNames": "public.ordersentry, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "TotalCount": 66, + "InvalidCount": 22, + "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -167,9 +167,11 @@ "test_views.view_table2", "test_views.mv1", "test_views.abc_mview", - "test_views.view_table1" + "test_views.view_table1", + "public.library_nested", + "public.orders_lateral" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 70 objects (62 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 72 objects (64 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -247,6 +249,18 @@ "TableName": "ordersentry_view", "ColumnName": "transaction_id", "DataType": "xid" + }, + { + "SchemaName": "public", + "TableName": "library_nested", + "ColumnName": "lib_data", + "DataType": "xml" + }, + { + "SchemaName": "public", + "TableName": "orders_lateral", + "ColumnName": "order_details", + "DataType": "xml" } ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", @@ -674,6 +688,34 @@ "ParentTableName": null, "SizeInBytes": 0 }, + { + "SchemaName": "public", + "ObjectName": "orders_lateral", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "library_nested", + "RowCount": 1, + "ColumnCount": 2, + "Reads": 1, + "Writes": 1, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, { "SchemaName": "public", "ObjectName": "orders2", @@ -2071,6 +2113,21 @@ "ConstructTypeName": "XML Functions", "Query": "SELECT xml_is_well_formed($1)", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "ConstructTypeName": "XML Functions", + "Query": "SELECT\n s.section_name,\n b.title,\n b.author\nFROM\n library_nested l,\n XMLTABLE(\n $1\n PASSING l.lib_data\n COLUMNS\n section_name TEXT PATH $2,\n books XML PATH $3\n ) AS s,\n XMLTABLE(\n $4\n PASSING s.books\n COLUMNS\n title TEXT PATH $5,\n author TEXT PATH $6\n) AS b", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "ConstructTypeName": "XML Functions", + "Query": "SELECT\n o.order_id,\n items.product,\n items.quantity::INT\nFROM\n orders_lateral o\n CROSS JOIN LATERAL XMLTABLE(\n $1\n PASSING o.order_details\n COLUMNS\n product TEXT PATH $2,\n quantity TEXT PATH $3\n) AS items", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + }, + { + "ConstructTypeName": "XML Functions", + "Query": "SELECT *\nFROM xmltable(\n $1\n PASSING $2\n COLUMNS \n name TEXT PATH $3\n)", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" } ], "UnsupportedPlPgSqlObjects": [ diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index 90d0e03d4e..70707a1fcb 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -25,19 +25,114 @@ SELECT xmlelement(name root, xmlelement(name child, 'value')); SELECT xml_is_well_formed('value'); --- Not Reported Currently +SELECT * +FROM xmltable( + '/employees/employee' + PASSING 'John' + COLUMNS + name TEXT PATH 'name' +); --- SELECT * --- FROM xmltable( --- '/employees/employee' --- PASSING 'John' --- COLUMNS --- name TEXT PATH 'name' --- ); -- Advisory Locks SELECT pg_advisory_lock(1,2); SELECT pg_advisory_unlock(1,2); SELECT pg_advisory_xact_lock(1,2); -SELECT pg_advisory_unlock_all(); \ No newline at end of file +SELECT pg_advisory_unlock_all(); + +-- Adding few XMLTABLE() examples +-- Case 1 +CREATE TABLE library_nested ( + lib_id INT, + lib_data XML +); + +INSERT INTO library_nested VALUES +(1, ' + +
    + + The Great Gatsby + F. Scott Fitzgerald + + + 1984 + George Orwell + +
    +
    + + A Brief History of Time + Stephen Hawking + +
    +     +'); + +-- Query with nested XMLTABLE() calls +SELECT + s.section_name, + b.title, + b.author +FROM + library_nested l, + XMLTABLE( + '/library/section' + PASSING l.lib_data + COLUMNS + section_name TEXT PATH '@name', + books XML PATH '.' + ) AS s, + XMLTABLE( + '/section/book' + PASSING s.books + COLUMNS + title TEXT PATH 'title', + author TEXT PATH 'author' +) AS b; + + +-- Case 2 +CREATE TABLE orders_lateral ( + order_id INT, + customer_id INT, + order_details XML +); + +INSERT INTO orders_lateral (customer_id, order_details) VALUES +(1, 1, ' + + + Keyboard + 2 + + + Mouse + 1 + + +'), +(2, ' + + + Monitor + 1 + + +'); + +-- Query using XMLTABLE with LATERAL join +SELECT + o.order_id, + items.product, + items.quantity::INT +FROM + orders_lateral o + CROSS JOIN LATERAL XMLTABLE( + '/order/item' + PASSING o.order_details + COLUMNS + product TEXT PATH 'product', + quantity TEXT PATH 'quantity' +) AS items; \ No newline at end of file diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 28b1b0ff4b..f63ee0e476 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -102,6 +102,7 @@ func (p *ParserIssueDetector) GetDMLIssues(query string) ([]issue.IssueInstance, NewFuncCallDetector(), NewColumnRefDetector(), NewXmlExprDetector(), + NewRangeTableFuncDetector(), } processor := func(msg protoreflect.Message) error { diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs.go b/yb-voyager/src/queryissue/unsupported_dml_constructs.go index 0077e699d3..2c06c0734e 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs.go +++ b/yb-voyager/src/queryissue/unsupported_dml_constructs.go @@ -111,3 +111,32 @@ func (d *XmlExprDetector) Detect(msg protoreflect.Message) ([]string, error) { } return nil, nil } + +/* +RangeTableFunc node manages functions that produce tables, structuring output into rows and columns +for SQL queries. Example: XMLTABLE() + +ASSUMPTION: +- RangeTableFunc is used for representing XMLTABLE() only as of now +- Comments from Postgres code: + - RangeTableFunc - raw form of "table functions" such as XMLTABLE + - Note: JSON_TABLE is also a "table function", but it uses JsonTable node, + - not RangeTableFunc. + +- link: https://github.com/postgres/postgres/blob/ea792bfd93ab8ad4ef4e3d1a741b8595db143677/src/include/nodes/parsenodes.h#L651 +*/ +type RangeTableFuncDetector struct{} + +func NewRangeTableFuncDetector() *RangeTableFuncDetector { + return &RangeTableFuncDetector{} +} + +// Detect checks if a RangeTableFunc node is present for a XMLTABLE() function +func (d *RangeTableFuncDetector) Detect(msg protoreflect.Message) ([]string, error) { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_RANGETABLEFUNC_NODE { + if queryparser.IsXMLTable(msg) { + return []string{XML_FUNCTIONS}, nil + } + } + return nil, nil +} diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go b/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go index d9974f1854..ba92994495 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go +++ b/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go @@ -20,7 +20,6 @@ import ( "sort" "testing" - pg_query "github.com/pganalyze/pg_query_go/v5" "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -29,7 +28,6 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" ) -// TestFuncCallDetector tests the Advisory Lock Detector. func TestFuncCallDetector(t *testing.T) { advisoryLockSqls := []string{ `SELECT pg_advisory_lock(100), COUNT(*) FROM cars;`, @@ -81,7 +79,7 @@ func TestFuncCallDetector(t *testing.T) { detector := NewFuncCallDetector() for _, sql := range advisoryLockSqls { - parseResult, err := pg_query.Parse(sql) + parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) visited := make(map[protoreflect.Message]bool) @@ -96,15 +94,13 @@ func TestFuncCallDetector(t *testing.T) { return nil } - parseTreeMsg := parseResult.Stmts[0].Stmt.ProtoReflect() + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - // The detector should detect Advisory Locks in these queries assert.Contains(t, unsupportedConstructs, ADVISORY_LOCKS, "Advisory Locks not detected in SQL: %s", sql) } } -// TestColumnRefDetector tests the System Column Detector. func TestColumnRefDetector(t *testing.T) { systemColumnSqls := []string{ `SELECT xmin, xmax FROM employees;`, @@ -147,9 +143,8 @@ func TestColumnRefDetector(t *testing.T) { } detector := NewColumnRefDetector() - for _, sql := range systemColumnSqls { - parseResult, err := pg_query.Parse(sql) + parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) visited := make(map[protoreflect.Message]bool) @@ -164,16 +159,199 @@ func TestColumnRefDetector(t *testing.T) { return nil } - parseTreeMsg := parseResult.Stmts[0].Stmt.ProtoReflect() + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - // The detector should detect System Columns in these queries assert.Contains(t, unsupportedConstructs, SYSTEM_COLUMNS, "System Columns not detected in SQL: %s", sql) } } -// TestXmlExprDetector tests the XML Function Detection. -func TestXmlExprDetectorAndFuncCallDetector(t *testing.T) { +func TestRangeTableFuncDetector(t *testing.T) { + xmlTableSqls := []string{ + // Test Case 1: Simple XMLTABLE usage with basic columns + `SELECT + p.id, + x.product_id, + x.product_name, + x.price + FROM + products_basic p, + XMLTABLE( + '//Product' + PASSING p.data + COLUMNS + product_id TEXT PATH 'ID', + product_name TEXT PATH 'Name', + price NUMERIC PATH 'Price' + ) AS x;`, + + // Test Case 2: XMLTABLE with CROSS JOIN LATERAL + `SELECT + o.order_id, + items.product, + items.quantity::INT + FROM + orders_lateral o + CROSS JOIN LATERAL XMLTABLE( + '/order/item' + PASSING o.order_details + COLUMNS + product TEXT PATH 'product', + quantity TEXT PATH 'quantity' + ) AS items;`, + + // Test Case 3: XMLTABLE within a Common Table Expression (CTE) + `WITH xml_data AS ( + SELECT id, xml_column FROM xml_documents_cte + ) + SELECT + xd.id, + e.emp_id, + e.name, + e.department + FROM + xml_data xd, + XMLTABLE( + '//Employee' + PASSING xd.xml_column + COLUMNS + emp_id INT PATH 'ID', + name TEXT PATH 'Name', + department TEXT PATH 'Department' + ) AS e;`, + + // Test Case 4: Nested XMLTABLEs for handling hierarchical XML structures + `SELECT + s.section_name, + b.title, + b.author + FROM + library_nested l, + XMLTABLE( + '/library/section' + PASSING l.lib_data + COLUMNS + section_name TEXT PATH '@name', + books XML PATH 'book' + ) AS s, + XMLTABLE( + '/book' + PASSING s.books + COLUMNS + title TEXT PATH 'title', + author TEXT PATH 'author' + ) AS b;`, + + // Test Case 5: XMLTABLE with XML namespaces + `SELECT + x.emp_name, + x.position, + x.city, + x.country + FROM + employees_ns, + XMLTABLE( + XMLNAMESPACES ( + 'http://example.com/emp' AS emp, + 'http://example.com/address' AS addr + ), + '/emp:Employee' -- Using the emp namespace prefix + PASSING employees_ns.emp_data + COLUMNS + emp_name TEXT PATH 'emp:Name', -- Using emp prefix + position TEXT PATH 'emp:Position', -- Using emp prefix + city TEXT PATH 'addr:Address/addr:City', + country TEXT PATH 'addr:Address/addr:Country' + ) AS x;`, + + // Test Case 6: XMLTABLE used within a VIEW creation + `CREATE VIEW order_items_view AS + SELECT + o.order_id, + o.customer_name, + items.product, + items.quantity::INT + FROM + orders_view o, + XMLTABLE( + '/order/item' + PASSING o.order_details + COLUMNS + product TEXT PATH 'product', + quantity TEXT PATH 'quantity' + ) AS items;`, + `CREATE VIEW public.order_items_view AS + SELECT o.order_id, + o.customer_name, + items.product, + (items.quantity)::integer AS quantity + FROM public.orders_view o, + LATERAL XMLTABLE(('/order/item'::text) PASSING (o.order_details) COLUMNS product text PATH ('product'::text), quantity text PATH ('quantity'::text)) items;`, + + // Test Case 7: XMLTABLE with aggregation functions + `SELECT + s.report_id, + SUM(t.amount::NUMERIC) AS total_sales + FROM + sales_reports_nested s, + XMLTABLE( + '/sales/transaction' + PASSING s.sales_data + COLUMNS + amount TEXT PATH 'amount' + ) AS t + GROUP BY + s.report_id;`, + + // Test Case 8: Nested XMLTABLE() with subqueries + `SELECT + s.report_id, + SUM(t.amount::NUMERIC) AS total_sales + FROM + sales_reports_complex s, + XMLTABLE( + '/sales/transaction' + PASSING s.sales_data + COLUMNS + transaction_id INT PATH '@id', + transaction_details XML PATH 'details' + ) AS t, + XMLTABLE( + '/transaction/detail' + PASSING t.transaction_details + COLUMNS + amount TEXT PATH 'amount' + ) AS detail + GROUP BY + s.report_id;`, + } + + detector := NewRangeTableFuncDetector() + for _, sql := range xmlTableSqls { + parseResult, err := queryparser.Parse(sql) + assert.NoError(t, err, "Failed to parse SQL: %s", sql) + + visited := make(map[protoreflect.Message]bool) + unsupportedConstructs := []string{} + + processor := func(msg protoreflect.Message) error { + constructs, err := detector.Detect(msg) + if err != nil { + return err + } + unsupportedConstructs = append(unsupportedConstructs, constructs...) + return nil + } + + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) + err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS, "XML Functions not detected in SQL: %s", sql) + } +} + +// TestXmlExprDetector tests the XML Function Detection - FuncCallDetector, XMLExprDetector, RangeTableFuncDetector. +func TestXMLFunctionsDetectors(t *testing.T) { xmlFunctionSqls := []string{ `SELECT id, xmlelement(name "employee", name) AS employee_data FROM employees;`, `SELECT id, xpath('/person/name/text()', data) AS name FROM xml_example;`, @@ -230,26 +408,25 @@ WHERE id = 5;`, `SELECT xmlforest(name, department) AS employee_info FROM employees WHERE id = 4;`, - // TODO: future - - // `SELECT xmltable.* - // FROM xmldata, - // XMLTABLE('//ROWS/ROW' - // PASSING data - // COLUMNS id int PATH '@id', - // ordinality FOR ORDINALITY, - // "COUNTRY_NAME" text, - // country_id text PATH 'COUNTRY_ID', - // size_sq_km float PATH 'SIZE[@unit = "sq_km"]', - // size_other text PATH - // 'concat(SIZE[@unit!="sq_km"], " ", SIZE[@unit!="sq_km"]/@unit)', - // premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified');`, - // `SELECT xmltable.* - // FROM XMLTABLE(XMLNAMESPACES('http://example.com/myns' AS x, - // 'http://example.com/b' AS "B"), - // '/x:example/x:item' - // PASSING (SELECT data FROM xmldata) - // COLUMNS foo int PATH '@foo', - // bar int PATH '@B:bar');`, + `SELECT xmltable.* + FROM xmldata, + XMLTABLE('//ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + ordinality FOR ORDINALITY, + "COUNTRY_NAME" text, + country_id text PATH 'COUNTRY_ID', + size_sq_km float PATH 'SIZE[@unit = "sq_km"]', + size_other text PATH + 'concat(SIZE[@unit!="sq_km"], " ", SIZE[@unit!="sq_km"]/@unit)', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified');`, + `SELECT xmltable.* + FROM XMLTABLE(XMLNAMESPACES('http://example.com/myns' AS x, + 'http://example.com/b' AS "B"), + '/x:example/x:item' + PASSING (SELECT data FROM xmldata) + COLUMNS foo int PATH '@foo', + bar int PATH '@B:bar');`, `SELECT xml_is_well_formed_content('Alpha') AS is_well_formed_content FROM projects WHERE project_id = 10;`, @@ -291,11 +468,12 @@ WHERE id = 1;`, detectors := []UnsupportedConstructDetector{ NewXmlExprDetector(), + NewRangeTableFuncDetector(), NewFuncCallDetector(), } for _, sql := range xmlFunctionSqls { - parseResult, err := pg_query.Parse(sql) + parseResult, err := queryparser.Parse(sql) assert.NoError(t, err) visited := make(map[protoreflect.Message]bool) @@ -314,7 +492,7 @@ WHERE id = 1;`, return nil } - parseTreeMsg := parseResult.Stmts[0].Stmt.ProtoReflect() + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) // The detector should detect XML Functions in these queries @@ -361,7 +539,7 @@ RETURNING id, NewXmlExprDetector(), } for _, sql := range combinationSqls { - parseResult, err := pg_query.Parse(sql) + parseResult, err := queryparser.Parse(sql) assert.NoError(t, err) visited := make(map[protoreflect.Message]bool) @@ -380,7 +558,7 @@ RETURNING id, return nil } - parseTreeMsg := parseResult.Stmts[0].Stmt.ProtoReflect() + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) diff --git a/yb-voyager/src/queryparser/helpers.go b/yb-voyager/src/queryparser/helpers.go index 7b57ddbbb0..d0c1cb57a4 100644 --- a/yb-voyager/src/queryparser/helpers.go +++ b/yb-voyager/src/queryparser/helpers.go @@ -15,7 +15,12 @@ limitations under the License. */ package queryparser -import "google.golang.org/protobuf/reflect/protoreflect" +import ( + "strings" + + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/reflect/protoreflect" +) const ( DOCS_LINK_PREFIX = "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/" @@ -79,12 +84,12 @@ func GetStringValueFromNode(nodeMsg protoreflect.Message) string { return "" } - // 'nodeMsg' is a 'pg_query.Node' getting the set field in the 'node' oneof - nodeField := nodeMsg.WhichOneof(nodeMsg.Descriptor().Oneofs().ByName("node")) + nodeField := getOneofActiveField(nodeMsg, "node") if nodeField == nil { return "" } + // Get the message corresponding to the set field nodeValue := nodeMsg.Get(nodeField) node := nodeValue.Message() if node == nil || !node.IsValid() { @@ -93,11 +98,16 @@ func GetStringValueFromNode(nodeMsg protoreflect.Message) string { nodeType := node.Descriptor().FullName() switch nodeType { + // Represents a simple string literal in a query, such as names or values directly provided in the SQL text. case PG_QUERY_STRING_NODE: - strField := node.Descriptor().Fields().ByName("sval") - strValue := node.Get(strField) - return strValue.String() - // example: SELECT * FROM employees; + return extractStringField(node, "sval") + // Represents a constant value in SQL expressions, often used for literal values like strings, numbers, or keywords. + case PG_QUERY_ACONST_NODE: + return extractAConstString(node) + // Represents a type casting operation in SQL, where a value is explicitly converted from one type to another using a cast expression. + case PG_QUERY_TYPECAST_NODE: + return traverseAndExtractAConst(node, "arg") + // Represents the asterisk '*' used in SQL to denote the selection of all columns in a table. Example: SELECT * FROM employees; case PG_QUERY_ASTAR_NODE: return "" default: @@ -105,6 +115,222 @@ func GetStringValueFromNode(nodeMsg protoreflect.Message) string { } } +// extractStringField safely extracts a string field from a node +// Sample example:: {column_ref:{fields:{string:{sval:"s"}} fields:{string:{sval:"tableoid"}} location:7} +func extractStringField(node protoreflect.Message, fieldName string) string { + strField := node.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if strField == nil || !node.Has(strField) { + return "" + } + return node.Get(strField).String() +} + +// extractAConstString extracts the string from an 'A_Const' node's 'sval' field +// Sample example:: rowexpr:{a_const:{sval:{sval:"//Product"} location:124}} +func extractAConstString(aConstMsg protoreflect.Message) string { + // Extract the 'sval' field from 'A_Const' + svalField := aConstMsg.Descriptor().Fields().ByName("sval") + if svalField == nil || !aConstMsg.Has(svalField) { + return "" + } + + svalMsg := aConstMsg.Get(svalField).Message() + if svalMsg == nil || !svalMsg.IsValid() { + return "" + } + + // Ensure svalMsg is of type 'pg_query.String' + if svalMsg.Descriptor().FullName() != "pg_query.String" { + return "" + } + + // Extract the actual string value from 'pg_query.String' + return extractStringField(svalMsg, "sval") +} + +// traverseAndExtractAConst traverses to a specified field and extracts the 'A_Const' string value +// Sample example:: rowexpr:{type_cast:{arg:{a_const:{sval:{sval:"/order/item"} +func traverseAndExtractAConst(nodeMsg protoreflect.Message, fieldName string) string { + field := nodeMsg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field == nil || !nodeMsg.Has(field) { + return "" + } + + childMsg := nodeMsg.Get(field).Message() + if childMsg == nil || !childMsg.IsValid() { + return "" + } + + return GetStringValueFromNode(childMsg) +} + +/* +Note: XMLTABLE() is not a simple function(stored in FuncCall node), its a table function +which generates set of rows using the info(about rows, columns, content) provided to it +Hence its requires a more complex node structure(RangeTableFunc node) to represent. + +XMLTABLE transforms XML data into relational table format, making it easier to query XML structures. +Detection in RangeTableFunc Node: +- docexpr: Refers to the XML data source, usually a column storing XML. +- rowexpr: XPath expression (starting with '/' or '//') defining the rows in the XML. +- columns: Specifies the data extraction from XML into relational columns. + +Example: Converting XML data about books into a table: +SQL Query: + + SELECT x.* + FROM XMLTABLE( + '/bookstore/book' + PASSING xml_column + COLUMNS + title TEXT PATH 'title', + author TEXT PATH 'author' + ) AS x; + +Parsetree: stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{string:{sval:"x"}} fields:{a_star:{}} location:7}} location:7}} +from_clause:{range_table_func: + {docexpr:{column_ref:{fields:{string:{sval:"xml_column"}} location:57}} + rowexpr:{a_const:{sval:{sval:"/bookstore/book"} location:29}} + columns:{range_table_func_col:{colname:"title" type_name:{names:{string:{sval:"text"}} typemod:-1 location:87} colexpr:{a_const:{sval:{sval:"title"} location:97}} location:81}} + columns:{range_table_func_col:{colname:"author" type_name:{names:{string:{sval:"text"}} typemod:-1 location:116} colexpr:{a_const:{sval:{sval:"author"} location:126}} location:109}} +alias:{aliasname:"x"} location:17}} limit_option:LIMIT_OPTION_DEFAULT op:SETOP_NONE}} stmt_len:142 + +Here, 'docexpr' points to 'xml_column' containing XML data, 'rowexpr' selects each 'book' node, and 'columns' extract 'title' and 'author' from each book. +Hence Presence of XPath in 'rowexpr' and structured 'columns' typically indicates XMLTABLE usage. +*/ +// Function to detect if a RangeTableFunc node represents XMLTABLE() +func IsXMLTable(rangeTableFunc protoreflect.Message) bool { + if GetMsgFullName(rangeTableFunc) != PG_QUERY_RANGETABLEFUNC_NODE { + return false + } + + log.Debug("checking if range table func node is for XMLTABLE()") + // Check for 'docexpr' field + docexprField := rangeTableFunc.Descriptor().Fields().ByName("docexpr") + if docexprField == nil { + return false + } + docexprNode := rangeTableFunc.Get(docexprField).Message() + if docexprNode == nil { + return false + } + + // Check for 'rowexpr' field + rowexprField := rangeTableFunc.Descriptor().Fields().ByName("rowexpr") + if rowexprField == nil { + return false + } + rowexprNode := rangeTableFunc.Get(rowexprField).Message() + if rowexprNode == nil { + return false + } + + xpath := GetStringValueFromNode(rowexprNode) + log.Debugf("xpath expression in the node: %s\n", xpath) + // Keep both cases check(param placeholder and absolute check) + if xpath == "" { + // Attempt to check if 'rowexpr' is a parameter placeholder like '$1' + isPlaceholder := IsParameterPlaceholder(rowexprNode) + if !isPlaceholder { + return false + } + } else if !IsXPathExprForXmlTable(xpath) { + return false + } + + // Check for 'columns' field + columnsField := rangeTableFunc.Descriptor().Fields().ByName("columns") + if columnsField == nil { + return false + } + + columnsList := rangeTableFunc.Get(columnsField).List() + if columnsList.Len() == 0 { + return false + } + + // this means all the required fields of RangeTableFunc node for being a XMLTABLE() are present + return true +} + +/* +isXPathExprForXmlTable checks whether a given string is a valid XPath expression for XMLTABLE()'s rowexpr. +It returns true if the expression starts with '/' or '//', indicating an absolute or anywhere path. +This covers the primary cases used in XMLTABLE() for selecting XML nodes as rows. + +XPath Expression Cases Covered for XMLTABLE(): +1. Absolute Paths: +- Starts with a single '/' indicating the root node. +- Example: "/library/book" + +2. Anywhere Paths: +- Starts with double '//' indicating selection from anywhere in the document. +- Example: "//book/author" + +For a comprehensive overview of XPath expressions, refer to: +https://developer.mozilla.org/en-US/docs/Web/XPath +*/ +func IsXPathExprForXmlTable(expression string) bool { + // Trim leading and trailing whitespace + expression = strings.TrimSpace(expression) + if expression == "" { + return false + } + + // Check if the expression starts with '/' or '//' + return strings.HasPrefix(expression, "/") || strings.HasPrefix(expression, "//") +} + func GetMsgFullName(msg protoreflect.Message) string { return string(msg.Descriptor().FullName()) } + +// IsParameterPlaceholder checks if the given node represents a parameter placeholder like $1 +func IsParameterPlaceholder(nodeMsg protoreflect.Message) bool { + if nodeMsg == nil || !nodeMsg.IsValid() { + return false + } + + nodeField := getOneofActiveField(nodeMsg, "node") + if nodeField == nil { + return false + } + + // Get the message corresponding to the set field + nodeValue := nodeMsg.Get(nodeField) + node := nodeValue.Message() + if node == nil || !node.IsValid() { + return false + } + + // Identify the type of the node + nodeType := node.Descriptor().FullName() + if nodeType == PG_QUERY_PARAMREF_NODE { + // This node represents a parameter reference like $1 + return true + } + + return false +} + +// getOneofActiveField retrieves the active field from a specified oneof in a Protobuf message. +// It returns the FieldDescriptor of the active field if a field is set, or nil if no field is set or the oneof is not found. +func getOneofActiveField(msg protoreflect.Message, oneofName string) protoreflect.FieldDescriptor { + if msg == nil { + return nil + } + + // Get the descriptor of the message and find the oneof by name + descriptor := msg.Descriptor() + if descriptor == nil { + return nil + } + + oneofDescriptor := descriptor.Oneofs().ByName(protoreflect.Name(oneofName)) + if oneofDescriptor == nil { + return nil + } + + // Determine which field within the oneof is set + return msg.WhichOneof(oneofDescriptor) +} diff --git a/yb-voyager/src/queryparser/traversal.go b/yb-voyager/src/queryparser/traversal.go index 090e30f055..1150e8f1b3 100644 --- a/yb-voyager/src/queryparser/traversal.go +++ b/yb-voyager/src/queryparser/traversal.go @@ -24,12 +24,16 @@ import ( ) const ( - PG_QUERY_NODE_NODE = "pg_query.Node" - PG_QUERY_STRING_NODE = "pg_query.String" - PG_QUERY_ASTAR_NODE = "pg_query.A_Star" - PG_QUERY_XMLEXPR_NODE = "pg_query.XmlExpr" - PG_QUERY_FUNCCALL_NODE = "pg_query.FuncCall" - PG_QUERY_COLUMNREF_NODE = "pg_query.ColumnRef" + PG_QUERY_NODE_NODE = "pg_query.Node" + PG_QUERY_STRING_NODE = "pg_query.String" + PG_QUERY_ASTAR_NODE = "pg_query.A_Star" + PG_QUERY_ACONST_NODE = "pg_query.A_Const" + PG_QUERY_TYPECAST_NODE = "pg_query.TypeCast" + PG_QUERY_XMLEXPR_NODE = "pg_query.XmlExpr" + PG_QUERY_FUNCCALL_NODE = "pg_query.FuncCall" + PG_QUERY_COLUMNREF_NODE = "pg_query.ColumnRef" + PG_QUERY_RANGETABLEFUNC_NODE = "pg_query.RangeTableFunc" + PG_QUERY_PARAMREF_NODE = "pg_query.ParamRef" ) // function type for processing nodes during traversal @@ -112,7 +116,7 @@ func TraverseParseTree(msg protoreflect.Message, visited map[protoreflect.Messag // Reference Oneof - https://protobuf.dev/programming-guides/proto3/#oneof if nodeType == PG_QUERY_NODE_NODE { - nodeField := msg.WhichOneof(msg.Descriptor().Oneofs().ByName("node")) + nodeField := getOneofActiveField(msg, "node") if nodeField != nil { value := msg.Get(nodeField) if value.IsValid() { diff --git a/yb-voyager/staticcheck.conf b/yb-voyager/staticcheck.conf new file mode 100644 index 0000000000..68d5a39312 --- /dev/null +++ b/yb-voyager/staticcheck.conf @@ -0,0 +1,2 @@ +# Disable the S1008 check to allow more explicit boolean returns +checks = ["-S1008"] \ No newline at end of file From 83d3b27fa30fd828dc0bdc2f724f78122311b617 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 21 Nov 2024 16:10:50 +0530 Subject: [PATCH 014/105] Partial fix for [export-data errors out if background metadata queries (count*) are still running when pg_dump completes.] (#1932) Implements fixes 1,3 in #1920 : When goroutines A,B,C,D encounter an error, they should not bring down the process, they should silently log the error and continue, as this is not a critical process. --- yb-voyager/cmd/exportDataStatus.go | 8 ++++++-- yb-voyager/src/srcdb/mysql.go | 8 ++++---- yb-voyager/src/srcdb/oracle.go | 8 ++++---- yb-voyager/src/srcdb/postgres.go | 8 ++++---- yb-voyager/src/srcdb/srcdb.go | 2 +- yb-voyager/src/srcdb/yugabytedb.go | 8 ++++---- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/yb-voyager/cmd/exportDataStatus.go b/yb-voyager/cmd/exportDataStatus.go index ee994bf0e0..e33468ae7b 100644 --- a/yb-voyager/cmd/exportDataStatus.go +++ b/yb-voyager/cmd/exportDataStatus.go @@ -102,7 +102,7 @@ func initializeExportTableMetadata(tableList []sqlname.NameTuple) { ExportedRowCountSnapshot: int64(0), } if source.DBType == POSTGRESQL { - //for Postgresql rename the table leaf table names to root table + //for Postgresql rename the table leaf table names to root table renamedTable, isRenamed := renameTableIfRequired(key) if isRenamed { exportSnapshotStatus.Tables[key].TableName = renamedTable @@ -203,7 +203,11 @@ func startExportPB(progressContainer *mpb.Progress, mapKey string, quitChan chan // parallel goroutine to calculate and set total to actual row count go func() { - actualRowCount := source.DB().GetTableRowCount(tableMetadata.TableName) + actualRowCount, err := source.DB().GetTableRowCount(tableMetadata.TableName) + if err != nil { + log.Warnf("could not get actual row count for table=%s: %v", tableMetadata.TableName, err) + return + } log.Infof("Replacing actualRowCount=%d inplace of expectedRowCount=%d for table=%s", actualRowCount, tableMetadata.CountTotalRows, tableMetadata.TableName.ForUserQuery()) pbr.SetTotalRowCount(actualRowCount, false) diff --git a/yb-voyager/src/srcdb/mysql.go b/yb-voyager/src/srcdb/mysql.go index 7bad4333b6..2c76207b5a 100644 --- a/yb-voyager/src/srcdb/mysql.go +++ b/yb-voyager/src/srcdb/mysql.go @@ -47,7 +47,7 @@ func newMySQL(s *Source) *MySQL { func (ms *MySQL) Connect() error { db, err := sql.Open("mysql", ms.getConnectionUri()) - db.SetMaxOpenConns(1) + db.SetMaxOpenConns(ms.source.NumConnections) db.SetConnMaxIdleTime(5 * time.Minute) ms.db = db return err @@ -75,17 +75,17 @@ func (ms *MySQL) CheckRequiredToolsAreInstalled() { checkTools("ora2pg") } -func (ms *MySQL) GetTableRowCount(tableName sqlname.NameTuple) int64 { +func (ms *MySQL) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.AsQualifiedCatalogName()) log.Infof("Querying row count of table %s", tableName) err := ms.db.QueryRow(query).Scan(&rowCount) if err != nil { - utils.ErrExit("Failed to query %q for row count of %q: %s", query, tableName, err) + return 0, fmt.Errorf("query %q for row count of %q: %w", query, tableName, err) } log.Infof("Table %q has %v rows.", tableName, rowCount) - return rowCount + return rowCount, nil } func (ms *MySQL) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 { diff --git a/yb-voyager/src/srcdb/oracle.go b/yb-voyager/src/srcdb/oracle.go index 47412fa709..b0b268d8e2 100644 --- a/yb-voyager/src/srcdb/oracle.go +++ b/yb-voyager/src/srcdb/oracle.go @@ -47,7 +47,7 @@ func newOracle(s *Source) *Oracle { func (ora *Oracle) Connect() error { db, err := sql.Open("godror", ora.getConnectionUri()) - db.SetMaxOpenConns(1) + db.SetMaxOpenConns(ora.source.NumConnections) db.SetConnMaxIdleTime(5 * time.Minute) ora.db = db return err @@ -82,17 +82,17 @@ func (ora *Oracle) CheckRequiredToolsAreInstalled() { checkTools("ora2pg", "sqlplus") } -func (ora *Oracle) GetTableRowCount(tableName sqlname.NameTuple) int64 { +func (ora *Oracle) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery()) log.Infof("Querying row count of table %q", tableName) err := ora.db.QueryRow(query).Scan(&rowCount) if err != nil { - utils.ErrExit("Failed to query %q for row count of %q: %s", query, tableName, err) + return 0, fmt.Errorf("query %q for row count of %q: %w", query, tableName, err) } log.Infof("Table %q has %v rows.", tableName, rowCount) - return rowCount + return rowCount, nil } func (ora *Oracle) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 { diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index b619d1f722..4037103e78 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -109,7 +109,7 @@ func newPostgreSQL(s *Source) *PostgreSQL { func (pg *PostgreSQL) Connect() error { db, err := sql.Open("pgx", pg.getConnectionUri()) - db.SetMaxOpenConns(1) + db.SetMaxOpenConns(pg.source.NumConnections) db.SetConnMaxIdleTime(5 * time.Minute) pg.db = db return err @@ -143,16 +143,16 @@ func (pg *PostgreSQL) CheckRequiredToolsAreInstalled() { checkTools("strings") } -func (pg *PostgreSQL) GetTableRowCount(tableName sqlname.NameTuple) int64 { +func (pg *PostgreSQL) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery()) log.Infof("Querying row count of table %q", tableName) err := pg.db.QueryRow(query).Scan(&rowCount) if err != nil { - utils.ErrExit("Failed to query %q for row count of %q: %s", query, tableName, err) + return 0, fmt.Errorf("query %q for row count of %q: %w", query, tableName, err) } log.Infof("Table %q has %v rows.", tableName, rowCount) - return rowCount + return rowCount, nil } func (pg *PostgreSQL) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 { diff --git a/yb-voyager/src/srcdb/srcdb.go b/yb-voyager/src/srcdb/srcdb.go index d16ac0d104..2d9c111e36 100644 --- a/yb-voyager/src/srcdb/srcdb.go +++ b/yb-voyager/src/srcdb/srcdb.go @@ -31,7 +31,7 @@ type SourceDB interface { Disconnect() CheckSchemaExists() bool GetConnectionUriWithoutPassword() string - GetTableRowCount(tableName sqlname.NameTuple) int64 + GetTableRowCount(tableName sqlname.NameTuple) (int64, error) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 CheckRequiredToolsAreInstalled() GetVersion() string diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index 1bc9256037..43022cf76e 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -52,7 +52,7 @@ func newYugabyteDB(s *Source) *YugabyteDB { func (yb *YugabyteDB) Connect() error { db, err := sql.Open("pgx", yb.getConnectionUri()) - db.SetMaxOpenConns(1) + db.SetMaxOpenConns(yb.source.NumConnections) db.SetConnMaxIdleTime(5 * time.Minute) yb.db = db return err @@ -74,16 +74,16 @@ func (yb *YugabyteDB) CheckRequiredToolsAreInstalled() { checkTools("strings") } -func (yb *YugabyteDB) GetTableRowCount(tableName sqlname.NameTuple) int64 { +func (yb *YugabyteDB) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery()) log.Infof("Querying row count of table %q", tableName) err := yb.db.QueryRow(query).Scan(&rowCount) if err != nil { - utils.ErrExit("Failed to query %q for row count of %q: %s", query, tableName, err) + return 0, fmt.Errorf("query %q for row count of %q: %w", query, tableName, err) } log.Infof("Table %q has %v rows.", tableName, rowCount) - return rowCount + return rowCount, nil } func (yb *YugabyteDB) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 { From 6804a0bb116ad5ba2b24a22996b253dc46535246 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 21 Nov 2024 17:28:07 +0530 Subject: [PATCH 015/105] Add truncate-tables flag to import-data-to-target (#1953) - --truncate-tables allows the user to truncate the tables before starting-clean. - Only applicable along with --start-clean true. - only implemented for import-data-to-target, not import-data-to-source-replica, import-data-file --- yb-voyager/cmd/import.go | 11 ++++++-- yb-voyager/cmd/importData.go | 29 ++++++++++++++++----- yb-voyager/cmd/root.go | 1 + yb-voyager/src/tgtdb/oracle.go | 4 +++ yb-voyager/src/tgtdb/postgres.go | 14 ++++++++++ yb-voyager/src/tgtdb/target_db_interface.go | 1 + yb-voyager/src/tgtdb/yugabytedb.go | 21 +++++++++++++++ 7 files changed, 72 insertions(+), 9 deletions(-) diff --git a/yb-voyager/cmd/import.go b/yb-voyager/cmd/import.go index 4f151166ee..983fd2ceb3 100644 --- a/yb-voyager/cmd/import.go +++ b/yb-voyager/cmd/import.go @@ -90,6 +90,7 @@ func validateImportFlags(cmd *cobra.Command, importerRole string) error { getSourceDBPassword(cmd) } validateParallelismFlags() + validateTruncateTablesFlag() return nil } @@ -231,13 +232,13 @@ func registerImportDataCommonFlags(cmd *cobra.Command) { cmd.Flags().MarkHidden("truncate-splits") } -func registerImportDataFlags(cmd *cobra.Command) { +func registerImportDataToTargetFlags(cmd *cobra.Command) { BoolVar(cmd.Flags(), &startClean, "start-clean", false, `Starts a fresh import with exported data files present in the export-dir/data directory. If any table on YugabyteDB database is non-empty, it prompts whether you want to continue the import without truncating those tables; If you go ahead without truncating, then yb-voyager starts ingesting the data present in the data files with upsert mode. Note that for the cases where a table doesn't have a primary key, this may lead to insertion of duplicate data. To avoid this, exclude the table using the --exclude-file-list or truncate those tables manually before using the start-clean flag (default false)`) - + BoolVar(cmd.Flags(), &truncateTables, "truncate-tables", false, "Truncate tables on target YugabyteDB before importing data. Only applicable along with --start-clean true (default false)") } func registerImportSchemaFlags(cmd *cobra.Command) { @@ -409,3 +410,9 @@ func validateParallelismFlags() { } } + +func validateTruncateTablesFlag() { + if truncateTables && !startClean { + utils.ErrExit("Error: --truncate-tables true can only be specified along with --start-clean true") + } +} diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index a69a543610..10c471fe2c 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -924,12 +924,27 @@ func cleanImportState(state *ImportDataState, tasks []*ImportFileTask) { nonEmptyTableNames := lo.Map(nonEmptyNts, func(nt sqlname.NameTuple, _ int) string { return nt.ForOutput() }) - utils.PrintAndLog("Non-Empty tables: [%s]", strings.Join(nonEmptyTableNames, ", ")) - utils.PrintAndLog("The above list of tables on target DB are not empty. ") - yes := utils.AskPrompt("Are you sure you want to start afresh without truncating tables") - if !yes { - utils.ErrExit("Aborting import. Manually truncate the tables on target DB before continuing.") + if importerRole == TARGET_DB_IMPORTER_ROLE && truncateTables { + // truncate tables only supported for import-data-to-target. + utils.PrintAndLog("Truncating non-empty tables on DB: %v", nonEmptyTableNames) + err := tdb.TruncateTables(nonEmptyNts) + if err != nil { + utils.ErrExit("failed to truncate tables: %s", err) + } + } else { + utils.PrintAndLog("Non-Empty tables: [%s]", strings.Join(nonEmptyTableNames, ", ")) + utils.PrintAndLog("The above list of tables on DB are not empty.") + if importerRole == TARGET_DB_IMPORTER_ROLE { + utils.PrintAndLog("If you wish to truncate them, re-run the import command with --truncate-tables true") + } else { + utils.PrintAndLog("If you wish to truncate them, re-run the import command after manually truncating the tables on DB.") + } + yes := utils.AskPrompt("Do you want to start afresh without truncating tables") + if !yes { + utils.ErrExit("Aborting import.") + } } + } for _, task := range tasks { @@ -1422,8 +1437,8 @@ func init() { registerTargetDBConnFlags(importDataToTargetCmd) registerImportDataCommonFlags(importDataCmd) registerImportDataCommonFlags(importDataToTargetCmd) - registerImportDataFlags(importDataCmd) - registerImportDataFlags(importDataToTargetCmd) + registerImportDataToTargetFlags(importDataCmd) + registerImportDataToTargetFlags(importDataToTargetCmd) } func createSnapshotImportStartedEvent() cp.SnapshotImportStartedEvent { diff --git a/yb-voyager/cmd/root.go b/yb-voyager/cmd/root.go index 35a09cfa58..7cca3f1b2f 100644 --- a/yb-voyager/cmd/root.go +++ b/yb-voyager/cmd/root.go @@ -45,6 +45,7 @@ var ( exportDir string schemaDir string startClean utils.BoolStr + truncateTables utils.BoolStr lockFile *lockfile.Lockfile migrationUUID uuid.UUID perfProfile utils.BoolStr diff --git a/yb-voyager/src/tgtdb/oracle.go b/yb-voyager/src/tgtdb/oracle.go index fa13b62da1..b66fbc986c 100644 --- a/yb-voyager/src/tgtdb/oracle.go +++ b/yb-voyager/src/tgtdb/oracle.go @@ -192,6 +192,10 @@ func (tdb *TargetOracleDB) GetNonEmptyTables(tables []sqlname.NameTuple) []sqlna return result } +func (tdb *TargetOracleDB) TruncateTables(tables []sqlname.NameTuple) error { + return fmt.Errorf("truncate tables not implemented for oracle") +} + func (tdb *TargetOracleDB) IsNonRetryableCopyError(err error) bool { return false } diff --git a/yb-voyager/src/tgtdb/postgres.go b/yb-voyager/src/tgtdb/postgres.go index f6c27223b2..92ac7ec1be 100644 --- a/yb-voyager/src/tgtdb/postgres.go +++ b/yb-voyager/src/tgtdb/postgres.go @@ -30,6 +30,7 @@ import ( "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" _ "github.com/jackc/pgx/v5/stdlib" + "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" @@ -331,6 +332,19 @@ func (pg *TargetPostgreSQL) GetNonEmptyTables(tables []sqlname.NameTuple) []sqln return result } +func (pg *TargetPostgreSQL) TruncateTables(tables []sqlname.NameTuple) error { + tableNames := lo.Map(tables, func(nt sqlname.NameTuple, _ int) string { + return nt.ForUserQuery() + }) + commaSeparatedTableNames := strings.Join(tableNames, ", ") + query := fmt.Sprintf("TRUNCATE TABLE %s", commaSeparatedTableNames) + _, err := pg.Exec(query) + if err != nil { + return fmt.Errorf("truncate tables with query %q: %w", query, err) + } + return nil +} + func (pg *TargetPostgreSQL) ImportBatch(batch Batch, args *ImportBatchArgs, exportDir string, tableSchema map[string]map[string]string) (int64, error) { var rowsAffected int64 var err error diff --git a/yb-voyager/src/tgtdb/target_db_interface.go b/yb-voyager/src/tgtdb/target_db_interface.go index 59e58404ed..faa3f75b8c 100644 --- a/yb-voyager/src/tgtdb/target_db_interface.go +++ b/yb-voyager/src/tgtdb/target_db_interface.go @@ -36,6 +36,7 @@ type TargetDB interface { GetVersion() string CreateVoyagerSchema() error GetNonEmptyTables(tableNames []sqlname.NameTuple) []sqlname.NameTuple + TruncateTables(tableNames []sqlname.NameTuple) error IsNonRetryableCopyError(err error) bool ImportBatch(batch Batch, args *ImportBatchArgs, exportDir string, tableSchema map[string]map[string]string) (int64, error) QuoteAttributeNames(tableNameTup sqlname.NameTuple, columns []string) ([]string, error) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 7683355afe..58f7f1b70a 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -33,7 +33,9 @@ import ( "github.com/google/uuid" "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" + pgconn5 "github.com/jackc/pgx/v5/pgconn" _ "github.com/jackc/pgx/v5/stdlib" + "github.com/samber/lo" log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" @@ -77,6 +79,12 @@ func (yb *TargetYugabyteDB) Exec(query string) (int64, error) { res, err := yb.db.Exec(query) if err != nil { + var pgErr *pgconn5.PgError + if errors.As(err, &pgErr) { + if pgErr.Hint != "" || pgErr.Detail != "" { + return rowsAffected, fmt.Errorf("run query %q on target %q: %w \nHINT: %s\nDETAIL: %s", query, yb.tconf.Host, err, pgErr.Hint, pgErr.Detail) + } + } return rowsAffected, fmt.Errorf("run query %q on target %q: %w", query, yb.tconf.Host, err) } rowsAffected, err = res.RowsAffected() @@ -397,6 +405,19 @@ func (yb *TargetYugabyteDB) GetNonEmptyTables(tables []sqlname.NameTuple) []sqln return result } +func (yb *TargetYugabyteDB) TruncateTables(tables []sqlname.NameTuple) error { + tableNames := lo.Map(tables, func(nt sqlname.NameTuple, _ int) string { + return nt.ForUserQuery() + }) + commaSeparatedTableNames := strings.Join(tableNames, ", ") + query := fmt.Sprintf("TRUNCATE TABLE %s", commaSeparatedTableNames) + _, err := yb.Exec(query) + if err != nil { + return err + } + return nil +} + func (yb *TargetYugabyteDB) ImportBatch(batch Batch, args *ImportBatchArgs, exportDir string, tableSchema map[string]map[string]string) (int64, error) { var rowsAffected int64 var err error From 6510471af60edff3d31595f469ce3a3bc909aeac Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:19:17 +0530 Subject: [PATCH 016/105] Using table list in guardrails queries related to checking permission of tables on the export side (#1955) - Building on the changes in PR: https://github.com/yugabyte/yb-voyager/pull/1824 - Added `queryTableList` instead of `querySchemaList` in all the guardrails queries related to tables on the export side - Removed 'ValidateTablesReadyForLiveMigration' since it was already getting covered in guardrails --- yb-voyager/cmd/assessMigrationCommand.go | 2 +- yb-voyager/cmd/exportData.go | 10 +- yb-voyager/cmd/exportSchema.go | 2 +- yb-voyager/cmd/importData.go | 22 +- yb-voyager/src/srcdb/mysql.go | 8 +- yb-voyager/src/srcdb/oracle.go | 8 +- yb-voyager/src/srcdb/postgres.go | 312 ++++++++++++----------- yb-voyager/src/srcdb/srcdb.go | 5 +- yb-voyager/src/srcdb/yugabytedb.go | 43 +++- 9 files changed, 214 insertions(+), 198 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 6bc2d2c8a8..f235d3b2a0 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -336,7 +336,7 @@ func assessMigration() (err error) { // Check if source db has permissions to assess migration if source.RunGuardrailsChecks { checkIfSchemasHaveUsagePermissions() - missingPerms, err := source.DB().GetMissingExportSchemaPermissions() + missingPerms, err := source.DB().GetMissingAssessMigrationPermissions() if err != nil { return fmt.Errorf("failed to get missing assess migration permissions: %w", err) } diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index 6040cf53ea..14cd85ab0f 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -261,7 +261,7 @@ func exportData() bool { // Check if source DB has required permissions for export data if source.RunGuardrailsChecks { - checkExportDataPermissions() + checkExportDataPermissions(finalTableList) } // finalize table list and column list @@ -348,10 +348,6 @@ func exportData() bool { // 2. export snapshot corresponding to replication slot by passing it to pg_dump // 3. start debezium with configration to read changes from the created replication slot, publication. - err := source.DB().ValidateTablesReadyForLiveMigration(finalTableList) - if err != nil { - utils.ErrExit("error: validate if tables are ready for live migration: %v", err) - } if !dataIsExported() { // if snapshot is not already done... err = exportPGSnapshotWithPGdump(ctx, cancel, finalTableList, tablesColumnList, leafPartitions) if err != nil { @@ -445,7 +441,7 @@ func exportData() bool { } } -func checkExportDataPermissions() { +func checkExportDataPermissions(finalTableList []sqlname.NameTuple) { // If source is PostgreSQL or YB, check if the number of existing replicaton slots is less than the max allowed if (source.DBType == POSTGRESQL && changeStreamingIsEnabled(exportType)) || (source.DBType == YUGABYTEDB && !bool(useYBgRPCConnector)) { @@ -463,7 +459,7 @@ func checkExportDataPermissions() { } } - missingPermissions, err := source.DB().GetMissingExportDataPermissions(exportType) + missingPermissions, err := source.DB().GetMissingExportDataPermissions(exportType, finalTableList) if err != nil { utils.ErrExit("get missing export data permissions: %v", err) } diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index b698654329..3625129955 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -135,7 +135,7 @@ func exportSchema() error { // Check if the source database has the required permissions for exporting schema. if source.RunGuardrailsChecks { checkIfSchemasHaveUsagePermissions() - missingPerms, err := source.DB().GetMissingExportSchemaPermissions() + missingPerms, err := source.DB().GetMissingExportSchemaPermissions("") if err != nil { return fmt.Errorf("failed to get missing migration permissions: %w", err) } diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 10c471fe2c..000c4d9519 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -217,11 +217,12 @@ func checkImportDataPermissions() { utils.PrintAndLog(output) var link string - if importerRole == SOURCE_REPLICA_DB_IMPORTER_ROLE { + switch importerRole { + case SOURCE_REPLICA_DB_IMPORTER_ROLE: link = "https://docs.yugabyte.com/preview/yugabyte-voyager/migrate/live-fall-forward/#prepare-source-replica-database" - } else if importerRole == SOURCE_DB_IMPORTER_ROLE { + case SOURCE_DB_IMPORTER_ROLE: link = "https://docs.yugabyte.com/preview/yugabyte-voyager/migrate/live-fall-back/#prepare-the-source-database" - } else { + default: if changeStreamingIsEnabled(importType) { link = "https://docs.yugabyte.com/preview/yugabyte-voyager/migrate/live-migrate/#prepare-the-target-database" } else { @@ -231,15 +232,18 @@ func checkImportDataPermissions() { fmt.Println("\nCheck the documentation to prepare the database for migration:", color.BlueString(link)) // Prompt user to continue if missing permissions only if fk and triggers check did not fail - if !fkAndTriggersCheckFailed { - if !utils.AskPrompt("\nDo you want to continue anyway") { - utils.ErrExit("Please grant the required permissions and retry the import.") - } - } else { + if fkAndTriggersCheckFailed { + utils.ErrExit("Please grant the required permissions and retry the import.") + } else if !utils.AskPrompt("\nDo you want to continue anyway") { utils.ErrExit("Please grant the required permissions and retry the import.") } } else { - log.Info("The target database has the required permissions for importing data.") + // If only fk and triggers check failed just simply error out + if fkAndTriggersCheckFailed { + utils.ErrExit("") + } else { + log.Info("The target database has the required permissions for importing data.") + } } } diff --git a/yb-voyager/src/srcdb/mysql.go b/yb-voyager/src/srcdb/mysql.go index 2c76207b5a..3cbec4f9a6 100644 --- a/yb-voyager/src/srcdb/mysql.go +++ b/yb-voyager/src/srcdb/mysql.go @@ -375,10 +375,6 @@ func (ms *MySQL) ParentTableOfPartition(table sqlname.NameTuple) string { panic("not implemented") } -func (ms *MySQL) ValidateTablesReadyForLiveMigration(tableList []sqlname.NameTuple) error { - panic("not implemented") -} - /* Only valid case is when the table has a auto increment column Note: a mysql table can have only one auto increment column @@ -534,11 +530,11 @@ func (ms *MySQL) CheckSourceDBVersion(exportType string) error { return nil } -func (ms *MySQL) GetMissingExportSchemaPermissions() ([]string, error) { +func (ms *MySQL) GetMissingExportSchemaPermissions(queryTableList string) ([]string, error) { return nil, nil } -func (ms *MySQL) GetMissingExportDataPermissions(exportType string) ([]string, error) { +func (ms *MySQL) GetMissingExportDataPermissions(exportType string, finalTableList []sqlname.NameTuple) ([]string, error) { return nil, nil } diff --git a/yb-voyager/src/srcdb/oracle.go b/yb-voyager/src/srcdb/oracle.go index b0b268d8e2..971b8993ed 100644 --- a/yb-voyager/src/srcdb/oracle.go +++ b/yb-voyager/src/srcdb/oracle.go @@ -434,10 +434,6 @@ func (ora *Oracle) ParentTableOfPartition(table sqlname.NameTuple) string { panic("not implemented") } -func (ora *Oracle) ValidateTablesReadyForLiveMigration(tableList []sqlname.NameTuple) error { - panic("not implemented") -} - /* GetColumnToSequenceMap returns a map of column name to sequence name for all identity columns in the given list of tables. Note: There can be only one identity column per table in Oracle @@ -717,11 +713,11 @@ func (ora *Oracle) CheckSourceDBVersion(exportType string) error { return nil } -func (ora *Oracle) GetMissingExportSchemaPermissions() ([]string, error) { +func (ora *Oracle) GetMissingExportSchemaPermissions(queryTableList string) ([]string, error) { return nil, nil } -func (ora *Oracle) GetMissingExportDataPermissions(exportType string) ([]string, error) { +func (ora *Oracle) GetMissingExportDataPermissions(exportType string, finalTableList []sqlname.NameTuple) ([]string, error) { return nil, nil } diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 4037103e78..c751b343de 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -225,10 +225,19 @@ func (pg *PostgreSQL) checkSchemasExists() []string { } func (pg *PostgreSQL) GetAllTableNamesRaw(schemaName string) ([]string, error) { - query := fmt.Sprintf(`SELECT table_name - FROM information_schema.tables - WHERE table_type = 'BASE TABLE' AND - table_schema = '%s';`, schemaName) + // Information schema requires select permission on the tables to query the tables. However, pg_catalog does not require any permission. + // So, we are using pg_catalog to get the table names. + query := fmt.Sprintf(` + SELECT + c.relname AS table_name + FROM + pg_catalog.pg_class c + JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE + c.relkind IN ('r', 'p') -- 'r' for regular tables, 'p' for partitioned tables + AND n.nspname = '%s'; + `, schemaName) rows, err := pg.db.Query(query) if err != nil { @@ -258,10 +267,20 @@ func (pg *PostgreSQL) GetAllTableNamesRaw(schemaName string) ([]string, error) { func (pg *PostgreSQL) GetAllTableNames() []*sqlname.SourceName { schemaList := pg.checkSchemasExists() querySchemaList := "'" + strings.Join(schemaList, "','") + "'" - query := fmt.Sprintf(`SELECT table_schema, table_name - FROM information_schema.tables - WHERE table_type = 'BASE TABLE' AND - table_schema IN (%s);`, querySchemaList) + // Information schema requires select permission on the tables to query the tables. However, pg_catalog does not require any permission. + // So, we are using pg_catalog to get the table names. + query := fmt.Sprintf(` + SELECT + n.nspname AS table_schema, + c.relname AS table_name + FROM + pg_catalog.pg_class c + JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE + c.relkind IN ('r', 'p') -- 'r' for regular tables, 'p' for partitioned tables + AND n.nspname IN (%s); + `, querySchemaList) rows, err := pg.db.Query(query) if err != nil { @@ -656,9 +675,17 @@ func (pg *PostgreSQL) ParentTableOfPartition(table sqlname.NameTuple) string { var parentTable string // For this query in case of case sensitive tables, minquoting is required - query := fmt.Sprintf(`SELECT inhparent::pg_catalog.regclass - FROM pg_catalog.pg_class c JOIN pg_catalog.pg_inherits ON c.oid = inhrelid - WHERE c.oid = '%s'::regclass::oid`, table.ForOutput()) + query := fmt.Sprintf(`SELECT + inhparent::pg_catalog.regclass AS parent_table + FROM + pg_catalog.pg_inherits + JOIN + pg_catalog.pg_class AS child ON pg_inherits.inhrelid = child.oid + JOIN + pg_catalog.pg_namespace AS nsp_child ON child.relnamespace = nsp_child.oid + WHERE + child.relname = '%s' + AND nsp_child.nspname = '%s';`, table.CurrentName.Unqualified.MinQuoted, table.CurrentName.SchemaName) err := pg.db.QueryRow(query).Scan(&parentTable) if err != sql.ErrNoRows && err != nil { @@ -948,43 +975,6 @@ func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { return nonPKTables, nil } -func (pg *PostgreSQL) ValidateTablesReadyForLiveMigration(tableList []sqlname.NameTuple) error { - var tablesWithReplicaIdentityNotFull []string - var qualifiedTableNames []string - for _, table := range tableList { - sname, tname := table.ForCatalogQuery() - qualifiedTableNames = append(qualifiedTableNames, fmt.Sprintf("'%s.%s'", sname, tname)) - } - query := fmt.Sprintf(`SELECT n.nspname || '.' || c.relname AS table_name_with_schema - FROM pg_class AS c - JOIN pg_namespace AS n ON c.relnamespace = n.oid - WHERE (n.nspname || '.' || c.relname) IN (%s) - AND c.relkind = 'r' - AND c.relreplident <> 'f';`, strings.Join(qualifiedTableNames, ",")) - rows, err := pg.db.Query(query) - if err != nil { - return fmt.Errorf("error in querying(%q) source database for replica identity: %v", query, err) - } - defer func() { - closeErr := rows.Close() - if closeErr != nil { - log.Warnf("close rows for query %q: %v", query, closeErr) - } - }() - for rows.Next() { - var tableWithSchema string - err := rows.Scan(&tableWithSchema) - if err != nil { - return fmt.Errorf("error in scanning query rows for replica identity: %v", err) - } - tablesWithReplicaIdentityNotFull = append(tablesWithReplicaIdentityNotFull, tableWithSchema) - } - if len(tablesWithReplicaIdentityNotFull) > 0 { - return fmt.Errorf("tables %v do not have REPLICA IDENTITY FULL\nPlease ALTER the tables and set their REPLICA IDENTITY to FULL", tablesWithReplicaIdentityNotFull) - } - return nil -} - // =============================== Guardrails =============================== func (pg *PostgreSQL) CheckSourceDBVersion(exportType string) error { @@ -1016,11 +1006,11 @@ Returns: - []string: A slice of strings describing the missing permissions, if any. - error: An error if any issues occur during the permission checks. */ -func (pg *PostgreSQL) GetMissingExportSchemaPermissions() ([]string, error) { +func (pg *PostgreSQL) GetMissingExportSchemaPermissions(queryTableList string) ([]string, error) { var combinedResult []string // Check if tables have SELECT permission - missingTables, err := pg.listTablesMissingSelectPermission() + missingTables, err := pg.listTablesMissingSelectPermission(queryTableList) if err != nil { return nil, fmt.Errorf("error checking table select permissions: %w", err) } @@ -1057,8 +1047,12 @@ The function performs the following checks: - Checks if the user has create permission on the database. - Checks if the user has ownership over all tables. */ -func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string) ([]string, error) { +func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string, finalTableList []sqlname.NameTuple) ([]string, error) { var combinedResult []string + qualifiedMinQuotedTableNames := lo.Map(finalTableList, func(table sqlname.NameTuple, _ int) string { + return table.ForOutput() + }) + queryTableList := fmt.Sprintf("'%s'", strings.Join(qualifiedMinQuotedTableNames, "','")) // For live migration if exportType == utils.CHANGES_ONLY || exportType == utils.SNAPSHOT_AND_CHANGES { @@ -1094,16 +1088,16 @@ func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string) ([]stri } // Check replica identity of tables - // missingTables, err := pg.listTablesMissingReplicaIdentityFull() - // if err != nil { - // return nil, fmt.Errorf("error in checking table replica identity: %w", err) - // } - // if len(missingTables) > 0 { - // combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString("Tables missing replica identity full: "), strings.Join(missingTables, ", "))) - // } + missingTables, err := pg.listTablesMissingReplicaIdentityFull(queryTableList) + if err != nil { + return nil, fmt.Errorf("error in checking table replica identity: %w", err) + } + if len(missingTables) > 0 { + combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString("Tables missing replica identity full: "), strings.Join(missingTables, ", "))) + } // Check if user has ownership over all tables - missingTables, err := pg.listTablesMissingOwnerPermission() + missingTables, err = pg.listTablesMissingOwnerPermission(queryTableList) if err != nil { return nil, fmt.Errorf("error in checking table owner permissions: %w", err) } @@ -1122,7 +1116,7 @@ func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string) ([]stri } else { // For offline migration // Check if schemas have USAGE permission and check if tables in the provided schemas have SELECT permission - res, err := pg.GetMissingExportSchemaPermissions() + res, err := pg.GetMissingExportSchemaPermissions(queryTableList) if err != nil { return nil, fmt.Errorf("error in getting missing export data permissions: %w", err) } @@ -1145,7 +1139,7 @@ func (pg *PostgreSQL) GetMissingAssessMigrationPermissions() ([]string, error) { var combinedResult []string // Check if tables have SELECT permission - missingTables, err := pg.listTablesMissingSelectPermission() + missingTables, err := pg.listTablesMissingSelectPermission("") if err != nil { return nil, fmt.Errorf("error checking table select permissions: %w", err) } @@ -1181,10 +1175,7 @@ func (pg *PostgreSQL) isMigrationUserASuperUser() (bool, error) { return isSuperUser, nil } -func (pg *PostgreSQL) listTablesMissingOwnerPermission() ([]string, error) { - trimmedSchemaList := pg.getTrimmedSchemaList() - querySchemaList := "'" + strings.Join(trimmedSchemaList, "','") + "'" - +func (pg *PostgreSQL) listTablesMissingOwnerPermission(queryTableList string) ([]string, error) { checkTableOwnerPermissionQuery := fmt.Sprintf(` WITH table_ownership AS ( SELECT @@ -1193,8 +1184,8 @@ func (pg *PostgreSQL) listTablesMissingOwnerPermission() ([]string, error) { pg_get_userbyid(c.relowner) AS owner_name FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid - WHERE c.relkind IN ('r', 'p') -- 'r' indicates a table 'p' indicates a partitioned table - AND n.nspname IN (%s) + WHERE c.relkind IN ('r', 'p') -- 'r' indicates a table, 'p' indicates a partitioned table + AND (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) IN (%s) ) SELECT schema_name, @@ -1211,7 +1202,7 @@ func (pg *PostgreSQL) listTablesMissingOwnerPermission() ([]string, error) { ) THEN true ELSE false END AS has_ownership - FROM table_ownership;`, querySchemaList, pg.source.User, pg.source.User) + FROM table_ownership;`, queryTableList, pg.source.User, pg.source.User) rows, err := pg.db.Query(checkTableOwnerPermissionQuery) if err != nil { @@ -1299,54 +1290,55 @@ func (pg *PostgreSQL) checkReplicationPermission() (bool, error) { return hasPermission, nil } -// func (pg *PostgreSQL) listTablesMissingReplicaIdentityFull() ([]string, error) { -// trimmedSchemaList := pg.getTrimmedSchemaList() -// querySchemaList := "'" + strings.Join(trimmedSchemaList, "','") + "'" -// checkTableReplicaIdentityQuery := fmt.Sprintf(`SELECT -// n.nspname AS schema_name, -// c.relname AS table_name, -// c.relreplident AS replica_identity, -// CASE -// WHEN c.relreplident <> 'f' -// THEN '%s' -// ELSE '%s' -// END AS status -// FROM pg_class c -// JOIN pg_namespace n ON c.relnamespace = n.oid -// WHERE quote_ident(n.nspname) IN (%s) -// AND c.relkind IN ('r', 'p');`, MISSING, GRANTED, querySchemaList) -// rows, err := pg.db.Query(checkTableReplicaIdentityQuery) -// if err != nil { -// return nil, fmt.Errorf("error in querying(%q) source database for checking table replica identity: %w", checkTableReplicaIdentityQuery, err) -// } -// defer func() { -// closeErr := rows.Close() -// if closeErr != nil { -// log.Warnf("close rows for query %q: %v", checkTableReplicaIdentityQuery, closeErr) -// } -// }() - -// var missingTables []string -// var tableSchemaName, tableName, replicaIdentity, status string - -// for rows.Next() { -// err = rows.Scan(&tableSchemaName, &tableName, &replicaIdentity, &status) -// if err != nil { -// return nil, fmt.Errorf("error in scanning query rows for table names: %w", err) -// } -// if status == MISSING { -// // quote table name as it can be case sensitive -// missingTables = append(missingTables, fmt.Sprintf(`%s."%s"`, tableSchemaName, tableName)) -// } -// } - -// // Check for errors during row iteration -// if err = rows.Err(); err != nil { -// return nil, fmt.Errorf("error iterating over query rows: %w", err) -// } - -// return missingTables, nil -// } +func (pg *PostgreSQL) listTablesMissingReplicaIdentityFull(queryTableList string) ([]string, error) { + checkTableReplicaIdentityQuery := fmt.Sprintf(` + SELECT + n.nspname AS schema_name, + c.relname AS table_name, + c.relreplident AS replica_identity, + CASE + WHEN c.relreplident <> 'f' + THEN '%s' + ELSE '%s' + END AS status + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) IN (%s) + AND c.relkind IN ('r', 'p'); + `, MISSING, GRANTED, queryTableList) + + rows, err := pg.db.Query(checkTableReplicaIdentityQuery) + if err != nil { + return nil, fmt.Errorf("error in querying(%q) source database for checking table replica identity: %w", checkTableReplicaIdentityQuery, err) + } + defer func() { + closeErr := rows.Close() + if closeErr != nil { + log.Warnf("close rows for query %q: %v", checkTableReplicaIdentityQuery, closeErr) + } + }() + + var missingTables []string + var tableSchemaName, tableName, replicaIdentity, status string + + for rows.Next() { + err = rows.Scan(&tableSchemaName, &tableName, &replicaIdentity, &status) + if err != nil { + return nil, fmt.Errorf("error in scanning query rows for table names: %w", err) + } + if status == MISSING { + // quote table name as it can be case sensitive + missingTables = append(missingTables, fmt.Sprintf(`%s."%s"`, tableSchemaName, tableName)) + } + } + + // Check for errors during row iteration + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over query rows: %w", err) + } + + return missingTables, nil +} func (pg *PostgreSQL) checkWalLevel() (msg string) { query := `SELECT current_setting('wal_level') AS wal_level;` @@ -1434,43 +1426,61 @@ func (pg *PostgreSQL) listSequencesMissingSelectPermission() (sequencesWithMissi return sequencesWithMissingPerm, nil } -func (pg *PostgreSQL) listTablesMissingSelectPermission() (tablesWithMissingPerm []string, err error) { +func (pg *PostgreSQL) listTablesMissingSelectPermission(queryTableList string) (tablesWithMissingPerm []string, err error) { // Users only need SELECT permissions on the tables of the schema they want to export for export schema - trimmedSchemaList := pg.getTrimmedSchemaList() - trimmedSchemaList = append(trimmedSchemaList, "pg_catalog", "information_schema") - querySchemaList := "'" + strings.Join(trimmedSchemaList, "','") + "'" + checkTableSelectPermissionQuery := "" + if queryTableList == "" { + + trimmedSchemaList := pg.getTrimmedSchemaList() + trimmedSchemaList = append(trimmedSchemaList, "pg_catalog", "information_schema") + querySchemaList := "'" + strings.Join(trimmedSchemaList, "','") + "'" + + checkTableSelectPermissionQuery = fmt.Sprintf(` + WITH schema_list AS ( + SELECT unnest(ARRAY[%s]) AS schema_name + ), + accessible_schemas AS ( + SELECT schema_name + FROM schema_list + WHERE has_schema_privilege('%s', quote_ident(schema_name), 'USAGE') + ) + SELECT + t.schemaname AS schema_name, + t.tablename AS table_name, + CASE + WHEN has_table_privilege('%s', quote_ident(t.schemaname) || '.' || quote_ident(t.tablename), 'SELECT') + THEN '%s' + ELSE '%s' + END AS status + FROM pg_tables t + JOIN accessible_schemas a ON t.schemaname = a.schema_name + UNION ALL + SELECT + t.schemaname AS schema_name, + t.tablename AS table_name, + '%s' AS status + FROM pg_tables t + WHERE t.schemaname IN (SELECT schema_name FROM schema_list) + AND NOT EXISTS ( + SELECT 1 + FROM accessible_schemas a + WHERE t.schemaname = a.schema_name + );`, querySchemaList, pg.source.User, pg.source.User, GRANTED, MISSING, NO_USAGE_PERMISSION) + } else { + checkTableSelectPermissionQuery = fmt.Sprintf(` + SELECT + t.schemaname AS schema_name, + t.tablename AS table_name, + CASE + WHEN has_table_privilege('%s', quote_ident(t.schemaname) || '.' || quote_ident(t.tablename), 'SELECT') + THEN '%s' + ELSE '%s' + END AS status + FROM pg_tables t + WHERE quote_ident(t.schemaname) || '.' || quote_ident(t.tablename) IN (%s); + `, pg.source.User, GRANTED, MISSING, queryTableList) + } - checkTableSelectPermissionQuery := fmt.Sprintf(` - WITH schema_list AS ( - SELECT unnest(ARRAY[%s]) AS schema_name - ), - accessible_schemas AS ( - SELECT schema_name - FROM schema_list - WHERE has_schema_privilege('%s', quote_ident(schema_name), 'USAGE') - ) - SELECT - t.schemaname AS schema_name, - t.tablename AS table_name, - CASE - WHEN has_table_privilege('%s', quote_ident(t.schemaname) || '.' || quote_ident(t.tablename), 'SELECT') - THEN '%s' - ELSE '%s' - END AS status - FROM pg_tables t - JOIN accessible_schemas a ON t.schemaname = a.schema_name - UNION ALL - SELECT - t.schemaname AS schema_name, - t.tablename AS table_name, - '%s' AS status - FROM pg_tables t - WHERE t.schemaname IN (SELECT schema_name FROM schema_list) - AND NOT EXISTS ( - SELECT 1 - FROM accessible_schemas a - WHERE t.schemaname = a.schema_name - );`, querySchemaList, pg.source.User, pg.source.User, GRANTED, MISSING, NO_USAGE_PERMISSION) rows, err := pg.db.Query(checkTableSelectPermissionQuery) if err != nil { return nil, fmt.Errorf("error in querying(%q) source database for checking table select permission: %w", checkTableSelectPermissionQuery, err) diff --git a/yb-voyager/src/srcdb/srcdb.go b/yb-voyager/src/srcdb/srcdb.go index 2d9c111e36..221fcac668 100644 --- a/yb-voyager/src/srcdb/srcdb.go +++ b/yb-voyager/src/srcdb/srcdb.go @@ -54,11 +54,10 @@ type SourceDB interface { GetTableToUniqueKeyColumnsMap(tableList []sqlname.NameTuple) (map[string][]string, error) ClearMigrationState(migrationUUID uuid.UUID, exportDir string) error GetNonPKTables() ([]string, error) - ValidateTablesReadyForLiveMigration(tableList []sqlname.NameTuple) error GetDatabaseSize() (int64, error) CheckSourceDBVersion(exportType string) error - GetMissingExportSchemaPermissions() ([]string, error) - GetMissingExportDataPermissions(exportType string) ([]string, error) + GetMissingExportSchemaPermissions(queryTableList string) ([]string, error) + GetMissingExportDataPermissions(exportType string, finalTableList []sqlname.NameTuple) ([]string, error) GetMissingAssessMigrationPermissions() ([]string, error) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) GetSchemasMissingUsagePermissions() ([]string, error) diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index 43022cf76e..025d5918ab 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -162,10 +162,19 @@ func (yb *YugabyteDB) checkSchemasExists() []string { } func (yb *YugabyteDB) GetAllTableNamesRaw(schemaName string) ([]string, error) { - query := fmt.Sprintf(`SELECT table_name - FROM information_schema.tables - WHERE table_type = 'BASE TABLE' AND - table_schema = '%s';`, schemaName) + // Information schema requires select permission on the tables to query the tables. However, pg_catalog does not require any permission. + // So, we are using pg_catalog to get the table names. + query := fmt.Sprintf(` + SELECT + c.relname AS table_name + FROM + pg_catalog.pg_class c + JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE + c.relkind IN ('r', 'p') -- 'r' for regular tables, 'p' for partitioned tables + AND n.nspname = '%s'; + `, schemaName) rows, err := yb.db.Query(query) if err != nil { @@ -195,10 +204,20 @@ func (yb *YugabyteDB) GetAllTableNamesRaw(schemaName string) ([]string, error) { func (yb *YugabyteDB) GetAllTableNames() []*sqlname.SourceName { schemaList := yb.checkSchemasExists() querySchemaList := "'" + strings.Join(schemaList, "','") + "'" - query := fmt.Sprintf(`SELECT table_schema, table_name - FROM information_schema.tables - WHERE table_type = 'BASE TABLE' AND - table_schema IN (%s);`, querySchemaList) + // Information schema requires select permission on the tables to query the tables. However, pg_catalog does not require any permission. + // So, we are using pg_catalog to get the table names. + query := fmt.Sprintf(` + SELECT + n.nspname AS table_schema, + c.relname AS table_name + FROM + pg_catalog.pg_class c + JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE + c.relkind IN ('r', 'p') -- 'r' for regular tables, 'p' for partitioned tables + AND n.nspname IN (%s); + `, querySchemaList) rows, err := yb.db.Query(query) if err != nil { @@ -264,10 +283,6 @@ func (yb *YugabyteDB) ExportSchema(exportDir string, schemaDir string) { panic("not implemented") } -func (yb *YugabyteDB) ValidateTablesReadyForLiveMigration(tableList []sqlname.NameTuple) error { - panic("not implemented") -} - func (yb *YugabyteDB) GetIndexesInfo() []utils.IndexInfo { return nil } @@ -1033,11 +1048,11 @@ func (yb *YugabyteDB) CheckSourceDBVersion(exportType string) error { return nil } -func (yb *YugabyteDB) GetMissingExportSchemaPermissions() ([]string, error) { +func (yb *YugabyteDB) GetMissingExportSchemaPermissions(queryTableList string) ([]string, error) { return nil, nil } -func (yb *YugabyteDB) GetMissingExportDataPermissions(exportType string) ([]string, error) { +func (yb *YugabyteDB) GetMissingExportDataPermissions(exportType string, finalTableList []sqlname.NameTuple) ([]string, error) { return nil, nil } From deeacdb0cb9dc05e26bb024432967e8ca48dedde Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 21 Nov 2024 20:00:20 +0530 Subject: [PATCH 017/105] Addded Guardrails check for pg_stat_statements in assess migration cmd (#1957) Note: checking if pgss is properly installed or not (via shared_preload_libraries) is not possible, needs pg_read_all_settings grant to the user. Felt unnecessary to add this to our grant script just for a check. So it will be best effort, if it able to fetch the libraries then we check in the list, otherwise just add warning to the logs and assess-migration command will fail at the step of db_queries_summary metadata script execution. --- yb-voyager/src/srcdb/postgres.go | 73 +++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index c751b343de..ec64b9179f 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -46,8 +46,9 @@ const MAX_SUPPORTED_PG_VERSION = "16" const MISSING = "MISSING" const GRANTED = "GRANTED" const NO_USAGE_PERMISSION = "NO USAGE PERMISSION" +const PG_STAT_STATEMENTS = "pg_stat_statements" -var pg_catalog_tables_required = []string{"regclass", "pg_class", "pg_inherits", "setval", "pg_index", "pg_relation_size", "pg_namespace", "pg_tables", "pg_sequences", "pg_roles", "pg_database"} +var pg_catalog_tables_required = []string{"regclass", "pg_class", "pg_inherits", "setval", "pg_index", "pg_relation_size", "pg_namespace", "pg_tables", "pg_sequences", "pg_roles", "pg_database", "pg_extension"} var information_schema_tables_required = []string{"schemata", "tables", "columns", "key_column_usage", "sequences"} var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID"} var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML"} @@ -1147,9 +1148,79 @@ func (pg *PostgreSQL) GetMissingAssessMigrationPermissions() ([]string, error) { combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString("Missing SELECT permission for user %s on Tables: ", pg.source.User), strings.Join(missingTables, ", "))) } + result, err := pg.checkPgStatStatementsSetup() + if err != nil { + return nil, fmt.Errorf("error checking pg_stat_statement extension installed with read permissions: %w", err) + } + + if result != "" { + combinedResult = append(combinedResult, result) + } return combinedResult, nil } +const ( + queryPgStatStatementsSchema = ` + SELECT nspname + FROM pg_extension e + JOIN pg_namespace n ON e.extnamespace = n.oid + WHERE e.extname = 'pg_stat_statements'` + + querySharedPreloadLibraries = `SELECT current_setting('shared_preload_libraries')` + + queryHasReadStatsPermission = ` + SELECT pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')` +) + +// checkPgStatStatementsSetup checks if pg_stat_statements is properly installed and if the user has the necessary read permissions. +func (pg *PostgreSQL) checkPgStatStatementsSetup() (string, error) { + if !utils.GetEnvAsBool("REPORT_UNSUPPORTED_QUERY_CONSTRUCTS", true) { + log.Infof("REPORT_UNSUPPORTED_QUERY_CONSTRUCTS is set as false, skipping guardrails check for pg_stat_statements") + return "", nil + } + + // 1. check if pg_stat_statements extension is available on source + var pgssExtSchema string + err := pg.db.QueryRow(queryPgStatStatementsSchema).Scan(&pgssExtSchema) + if err != nil && err != sql.ErrNoRows { + return "", fmt.Errorf("failed to fetch the schema of pg_stat_statement available in: %w", err) + } + if pgssExtSchema == "" { + return "pg_stat_statements extension is not installed on source DB, required for detecting Unsupported Query Constructs", nil + } else { + schemaList := lo.Union(pg.getTrimmedSchemaList(), []string{"public"}) + if !slices.Contains(schemaList, pgssExtSchema) { + return fmt.Sprintf("pg_stat_statements extension schema %q is not in the schema list (%s), required for detecting Unsupported Query Constructs", + pgssExtSchema, strings.Join(schemaList, ", ")), nil + } + } + + // 2. check if its properly installed/loaded + // To access "shared_preload_libraries" must be superuser or a member of pg_read_all_settings + // so trying a best effort here, if accessible then check otherwise it will fail during the gather metadata + var sharedPreloadLibraries string + err = pg.db.QueryRow(querySharedPreloadLibraries).Scan(&sharedPreloadLibraries) + if err != nil { + log.Warnf("failed to check if pg_stat_statements extension is properly loaded on source DB: %v", err) + } + if !slices.Contains(strings.Split(sharedPreloadLibraries, ","), PG_STAT_STATEMENTS) { + return "pg_stat_statements is not loaded via shared_preload_libraries, required for detecting Unsupported Query Constructs", nil + } + + // 3. User has permission to read from pg_stat_statements table + var hasReadAllStats bool + err = pg.db.QueryRow(queryHasReadStatsPermission).Scan(&hasReadAllStats) + if err != nil { + return "", fmt.Errorf("failed to check pg_read_all_stats grant on migration user: %w", err) + } + + if !hasReadAllStats { + return "User doesn't have permissions to read pg_stat_statements view, unsupported query constructs won't be detected/reported", nil + } + + return "", nil +} + func (pg *PostgreSQL) isMigrationUserASuperUser() (bool, error) { query := ` SELECT From aa0756015c2fc8ab994079e23c1576d6faef93c4 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 22 Nov 2024 17:27:20 +0530 Subject: [PATCH 018/105] Fix: export data status command should not write in name registry (#1964) https://yugabyte.atlassian.net/browse/DB-14187 --- yb-voyager/cmd/exportDataStatusCommand.go | 8 +++++--- yb-voyager/src/namereg/namereg.go | 6 +++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/yb-voyager/cmd/exportDataStatusCommand.go b/yb-voyager/cmd/exportDataStatusCommand.go index af948a0cf4..78c9697b5e 100644 --- a/yb-voyager/cmd/exportDataStatusCommand.go +++ b/yb-voyager/cmd/exportDataStatusCommand.go @@ -55,7 +55,7 @@ var exportDataStatusCmd = &cobra.Command{ utils.ErrExit("\nNote: Run the following command to get the current report of live migration:\n" + color.CyanString("yb-voyager get data-migration-report --export-dir %q\n", exportDir)) } - err = InitNameRegistry(exportDir, SOURCE_DB_EXPORTER_ROLE, nil, nil, nil, nil, false) + err = InitNameRegistry(exportDir, namereg.SOURCE_DB_EXPORTER_STATUS_ROLE, nil, nil, nil, nil, false) if err != nil { utils.ErrExit("initializing name registry: %v", err) } @@ -69,7 +69,9 @@ var exportDataStatusCmd = &cobra.Command{ } useDebezium = msr.IsSnapshotExportedViaDebezium() - + if msr.SourceDBConf == nil { + utils.ErrExit("export data has not started yet. Try running after export has started") + } source = *msr.SourceDBConf sqlname.SourceDBType = source.DBType leafPartitions := getLeafPartitionsFromRootTable() @@ -162,7 +164,7 @@ func getSnapshotExportStatusRow(tableStatus *dbzm.TableExportStatus, leafPartiti func getDisplayName(nt sqlname.NameTuple, partitions []string, isTableListSet bool) string { displayTableName := nt.ForMinOutput() //Changing the display of the partition tables in case table-list is set because there can be case where user has passed a subset of leaft tables in the list - if source.DBType == POSTGRESQL && partitions != nil && isTableListSet { + if source.DBType == POSTGRESQL && partitions != nil && isTableListSet { slices.Sort(partitions) partitions := strings.Join(partitions, ", ") displayTableName = fmt.Sprintf("%s (%s)", displayTableName, partitions) diff --git a/yb-voyager/src/namereg/namereg.go b/yb-voyager/src/namereg/namereg.go index 12ff9045cb..5aebe6380e 100644 --- a/yb-voyager/src/namereg/namereg.go +++ b/yb-voyager/src/namereg/namereg.go @@ -25,6 +25,7 @@ const ( SOURCE_DB_IMPORTER_ROLE = "source_db_importer" // Fallback. SOURCE_REPLICA_DB_IMPORTER_ROLE = "source_replica_db_importer" // Fall-forward. SOURCE_DB_EXPORTER_ROLE = "source_db_exporter" + SOURCE_DB_EXPORTER_STATUS_ROLE = "source_db_exporter_status" TARGET_DB_EXPORTER_FF_ROLE = "target_db_exporter_ff" TARGET_DB_EXPORTER_FB_ROLE = "target_db_exporter_fb" IMPORT_FILE_ROLE = "import_file" @@ -147,6 +148,9 @@ func (reg *NameRegistry) UnRegisterYBNames() error { } func (reg *NameRegistry) registerSourceNames() (bool, error) { + if reg.params.SDB == nil { + return false, fmt.Errorf("source db connection is not available") + } reg.SourceDBType = reg.params.SourceDBType reg.initSourceDBSchemaNames() m := make(map[string][]string) @@ -403,7 +407,7 @@ func NewNameTuple(role string, sourceName *sqlname.ObjectName, targetName *sqlna t.CurrentName = t.SourceName case SOURCE_REPLICA_DB_IMPORTER_ROLE: t.CurrentName = t.SourceName - case SOURCE_DB_EXPORTER_ROLE: + case SOURCE_DB_EXPORTER_ROLE, SOURCE_DB_EXPORTER_STATUS_ROLE: t.CurrentName = t.SourceName case TARGET_DB_EXPORTER_FF_ROLE, TARGET_DB_EXPORTER_FB_ROLE: t.CurrentName = t.TargetName From 760bb88ab8c89458c572edefa99d4ef17f1d9d3b Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 25 Nov 2024 17:30:52 +0530 Subject: [PATCH 019/105] Removed check for shared_preload_libraries from assess-migration guardrails check (#1970) - added a TODO for later --- yb-voyager/src/srcdb/postgres.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index ec64b9179f..839871df5b 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -1195,17 +1195,19 @@ func (pg *PostgreSQL) checkPgStatStatementsSetup() (string, error) { } } + // TODO: finalise the approach for detecting shared_preload_libraries // 2. check if its properly installed/loaded // To access "shared_preload_libraries" must be superuser or a member of pg_read_all_settings // so trying a best effort here, if accessible then check otherwise it will fail during the gather metadata - var sharedPreloadLibraries string - err = pg.db.QueryRow(querySharedPreloadLibraries).Scan(&sharedPreloadLibraries) - if err != nil { - log.Warnf("failed to check if pg_stat_statements extension is properly loaded on source DB: %v", err) - } - if !slices.Contains(strings.Split(sharedPreloadLibraries, ","), PG_STAT_STATEMENTS) { - return "pg_stat_statements is not loaded via shared_preload_libraries, required for detecting Unsupported Query Constructs", nil - } + // var sharedPreloadLibraries string + // err = pg.db.QueryRow(querySharedPreloadLibraries).Scan(&sharedPreloadLibraries) + // if err != nil { + // log.Warnf("failed to check if pg_stat_statements extension is properly loaded on source DB: %v", err) + // } else { + // if !slices.Contains(strings.Split(sharedPreloadLibraries, ","), PG_STAT_STATEMENTS) { + // return "pg_stat_statements is not loaded via shared_preload_libraries, required for detecting Unsupported Query Constructs", nil + // } + // } // 3. User has permission to read from pg_stat_statements table var hasReadAllStats bool From 2715c277cca7f3ca5636e319e5a8d9c28c083c10 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 27 Nov 2024 23:31:48 +0530 Subject: [PATCH 020/105] Fix: template rendering extra spaces after Unsupported PL/pgSQL objects section heading (#1996) --- .../migration_assessment_report.template | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index bd949d964b..c172979362 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -303,38 +303,38 @@ Statement Details - {{ range .UnsupportedPlPgSqlObjects }} - - - {{ $objectsGroupByObjectType := groupByObjectType .Objects}} - {{ $numUniqueObjectNamesOfAllTypes := totalUniqueObjectNamesOfAllTypes $objectsGroupByObjectType}} + {{ range .UnsupportedPlPgSqlObjects }} + + + {{ $objectsGroupByObjectType := groupByObjectType .Objects }} + {{ $numUniqueObjectNamesOfAllTypes := totalUniqueObjectNamesOfAllTypes $objectsGroupByObjectType }} {{ $docsLink := .DocsLink }} - {{ .FeatureName }} + {{ .FeatureName }} {{ $isNextRowRequiredForObjectType := false }} {{ range $type, $objectsByType := $objectsGroupByObjectType }} {{ $objectGroupByObjectName := groupByObjectName $objectsByType }} {{ $numUniqueObjectNames := numKeysInMapStringObjectInfo $objectGroupByObjectName }} {{ if $isNextRowRequiredForObjectType }} - + {{ end }} - {{$type}} + {{ $type }} {{ $isNextRowRequiredForObjectName := false }} {{ range $name, $objectsByName := $objectGroupByObjectName }} {{ if $isNextRowRequiredForObjectName }} - + {{ end }} - {{ $name }}  + {{ $name }}
      - {{ range $objectsByName }} - 
    • {{ .SqlStatement }}
      • - {{ end }} + {{ range $objectsByName }} + 
    • {{ .SqlStatement }}
      • + {{ end }}
    {{ if not $isNextRowRequiredForObjectType }} - Link + Link {{ end }} {{ $isNextRowRequiredForObjectName = true }} {{ $isNextRowRequiredForObjectType = true }} From 8731fc14996c5cc5fc8b0ccee123a2ab10ed1223 Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Thu, 28 Nov 2024 15:04:29 +0530 Subject: [PATCH 021/105] Removed start clean from the automation commands (#2005) * Removed start clean from the automation commands * Fix for schema migration test * Added a comment * Kept start-clean in import data file --- migtests/scripts/functions.sh | 6 ------ migtests/scripts/run-schema-migration.sh | 3 ++- migtests/tests/import-file/run-import-file-test | 8 +++++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 9059bea7c4..5b10574cc3 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -236,7 +236,6 @@ export_schema() { --source-db-password ${SOURCE_DB_PASSWORD} --source-db-name ${SOURCE_DB_NAME} --send-diagnostics=false --yes - --start-clean t " if [ "${source_db_schema}" != "" ] then @@ -280,7 +279,6 @@ export_data() { --disable-pb=true --send-diagnostics=false --yes - --start-clean 1 " if [ "${TABLE_LIST}" != "" ] then @@ -371,7 +369,6 @@ import_schema() { --target-db-name ${TARGET_DB_NAME} --yes --send-diagnostics=false - --start-clean 1 " if [ "${SOURCE_DB_TYPE}" != "postgresql" ] @@ -392,7 +389,6 @@ import_data() { --target-db-name ${TARGET_DB_NAME} --disable-pb true --send-diagnostics=false - --start-clean 1 --truncate-splits true --max-retries 1 " @@ -436,7 +432,6 @@ import_data_to_source_replica() { --source-replica-db-user ${SOURCE_REPLICA_DB_USER} --source-replica-db-name ${SOURCE_REPLICA_DB_NAME} --source-replica-db-password ${SOURCE_REPLICA_DB_PASSWORD} - --start-clean true --disable-pb true --send-diagnostics=false --parallel-jobs 3 @@ -762,7 +757,6 @@ assess_migration() { --source-db-password ${SOURCE_DB_PASSWORD} --source-db-name ${SOURCE_DB_NAME} --send-diagnostics=false --yes - --start-clean t --iops-capture-interval 0 " if [ "${SOURCE_DB_SCHEMA}" != "" ] diff --git a/migtests/scripts/run-schema-migration.sh b/migtests/scripts/run-schema-migration.sh index 9dfce9311f..0f37bab501 100755 --- a/migtests/scripts/run-schema-migration.sh +++ b/migtests/scripts/run-schema-migration.sh @@ -118,7 +118,8 @@ main() { mv "${EXPORT_DIR}/schema/failed.sql" "${EXPORT_DIR}/schema/failed.sql.bak" #replace_files replace_files "${TEST_DIR}/replacement_dir" "${EXPORT_DIR}/schema" - import_schema + # --start-clean is required here since we are running the import command for the second time + import_schema --start-clean t if [ -f "${EXPORT_DIR}/schema/failed.sql" ] then diff --git a/migtests/tests/import-file/run-import-file-test b/migtests/tests/import-file/run-import-file-test index 93303f04c3..2a8bf0eeaa 100755 --- a/migtests/tests/import-file/run-import-file-test +++ b/migtests/tests/import-file/run-import-file-test @@ -41,9 +41,11 @@ main() { export TARGET_DB_SCHEMA='non_public' +# Kept --start-clean here to test the command for import data file. Can add proper validations for it once --truncate-tables is introduced here + step "Import data file: SMSA.txt -> smsa in a non-public schema" import_data_file --data-dir ${TEST_DIR} --format text --delimiter '\t' \ - --file-table-map "SMSA.txt:smsa" --start-clean true + --file-table-map "SMSA.txt:smsa" --start-clean true #clean up the export dir as we will have public schema from this test which should be on fresh export-dir rm -rf ${EXPORT_DIR} @@ -53,7 +55,7 @@ main() { #Default BATCH_SIZE_BYTES step "Import data file: OneMRows.text -> one_m_rows" import_data_file --data-dir ${TEST_DIR} --format text --delimiter '|' \ - --file-table-map "OneMRows.text:one_m_rows" --start-clean true + --file-table-map "OneMRows.text:one_m_rows" export MAX_BATCH_SIZE_BYTES=345643 #~300KB @@ -276,7 +278,7 @@ main() { if [ "${RUN_LARGE_IMPORT_DATA_FILE_TEST}" = true ] ; then step "Run large sized import data file test" - import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts_large" --start-clean true --yes + import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts_large" --yes fi From 1507df2ab3dc3ce338bc86bdfa0ac3a6e1558135 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 28 Nov 2024 17:37:48 +0530 Subject: [PATCH 022/105] Update default Debezium Server HTTP port to use random freeport instead of 8080 (#1691) - Fetching free port dynamically by connecting to Port 0. Generally it returns 5 digit port numbers so there shouldn't be conflicts with standard ports like 5432, 8080 etc --- yb-voyager/src/dbzm/config.go | 14 ++++++++++++++ yb-voyager/src/utils/utils.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index 9a12d484eb..b8b2909e6f 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -81,6 +81,7 @@ type Config struct { var baseConfigTemplate = ` debezium.format.value=connect debezium.format.key=connect +quarkus.http.port=%d quarkus.log.console.json=false quarkus.log.level=%s ` @@ -295,10 +296,19 @@ func (c *Config) String() string { } else { log.Infof("QUEUE_SEGMENT_MAX_BYTES: %d", queueSegmentMaxBytes) } + + quarkusLogPort, err := utils.GetFreePort() + if err != nil { + log.Warnf("failed to get a free port for quarkus http server, falling back to 8080: %v", err) + quarkusLogPort = 8080 + } + log.Infof("using port number %d for quarkus http server", quarkusLogPort) + var conf string switch c.SourceDBType { case "postgresql": conf = fmt.Sprintf(postgresConfigTemplate, + quarkusLogPort, c.LogLevel, c.Username, c.SnapshotMode, @@ -332,6 +342,7 @@ func (c *Config) String() string { case "yugabytedb": if !c.UseYBgRPCConnector { conf = fmt.Sprintf(yugabyteLogicalReplicationConfigTemplate, + quarkusLogPort, c.LogLevel, c.Username, "never", @@ -359,6 +370,7 @@ func (c *Config) String() string { } } else { conf = fmt.Sprintf(yugabyteConfigTemplate, + quarkusLogPort, c.LogLevel, c.Username, "never", @@ -391,6 +403,7 @@ func (c *Config) String() string { } case "oracle": conf = fmt.Sprintf(oracleConfigTemplate, + quarkusLogPort, c.LogLevel, c.Username, c.SnapshotMode, @@ -422,6 +435,7 @@ func (c *Config) String() string { case "mysql": conf = fmt.Sprintf(mysqlConfigTemplate, + quarkusLogPort, c.LogLevel, c.Username, c.SnapshotMode, diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index 2bb1a9bd4c..0d105415c2 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -684,3 +684,17 @@ func ChangeFileExtension(filePath string, newExt string) string { return filePath + newExt } + +// Port 0 generally returns port number in range 30xxx - 60xxx but it also depends on OS and network configuration +func GetFreePort() (int, error) { + // Listen on port 0, which tells the OS to assign an available port + listener, err := net.Listen("tcp", ":0") + if err != nil { + return 0, fmt.Errorf("failed to listen on a port: %v", err) + } + defer listener.Close() + + // Retrieve the assigned port + addr := listener.Addr().(*net.TCPAddr) + return addr.Port, nil +} From 484ba666d0038909bda805922915e5eac0e17318 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:20:09 +0530 Subject: [PATCH 023/105] Added a new const PREVIOUS_BREAKING_CHANGE_VERSION and implemented version compatibility checks using it (#1966) - Added a new constant `PREVIOUS_BREAKING_CHANGE_VERSION` and set it to `1.8.5` for now. This needs to be changed every time we make a breaking release. - Added logic to check the Voyager version used to create the export directory against the breaking change version. - Ensured compatibility checks handle scenarios where the export directory was created using: - The `main` branch. - An older Voyager version without version information in msr. - A specific Voyager version. --- yb-voyager/cmd/common.go | 62 ++++++++++++++++++++++++++++----- yb-voyager/src/utils/utils.go | 20 +++++++++++ yb-voyager/src/utils/version.go | 3 ++ 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 112e6fdeb2..df1cb4f25e 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -46,6 +46,7 @@ import ( "golang.org/x/exp/slices" "golang.org/x/term" + "github.com/hashicorp/go-version" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" "github.com/yugabyte/yb-voyager/yb-voyager/src/datafile" @@ -475,22 +476,65 @@ func initMetaDB(migrationExportDir string) *metadb.MetaDB { if err != nil { utils.ErrExit("could not init migration status record: %w", err) } + msr, err := metaDBInstance.GetMigrationStatusRecord() if err != nil { utils.ErrExit("get migration status record: %v", err) } - if msr.VoyagerVersion != utils.YB_VOYAGER_VERSION { - userFacingMsg := fmt.Sprintf("Voyager requires the entire migration workflow to be executed using a single Voyager version.\n"+ - "The export-dir %q was created using version %q and the current version is %q. Either use Voyager %q to continue the migration or start afresh "+ - "with a new export-dir.", migrationExportDir, msr.VoyagerVersion, utils.YB_VOYAGER_VERSION, msr.VoyagerVersion) - if msr.VoyagerVersion == "" { //In case the export dir is already started from older version that will not have VoyagerVersion field in MSR - userFacingMsg = fmt.Sprintf("Voyager requires the entire migration workflow to be executed using a single Voyager version.\n"+ - "The export-dir %q was created using older version and the current version is %q. Either use older version to continue the migration or start afresh "+ - "with a new export-dir.", migrationExportDir, utils.YB_VOYAGER_VERSION) + + msrVoyagerVersionString := msr.VoyagerVersion + + detectVersionCompatibility(msrVoyagerVersionString, migrationExportDir) + return metaDBInstance +} + +func detectVersionCompatibility(msrVoyagerVersionString string, migrationExportDir string) { + // If the msr VoyagerVersion is less than the PREVIOUS_BREAKING_CHANGE_VERSION, then the export-dir is not compatible with the current Voyager version. + // This version will always be a final release version and never "main" or "rc" version. + previousBreakingChangeVersion, err := version.NewVersion(utils.PREVIOUS_BREAKING_CHANGE_VERSION) + if err != nil { + utils.ErrExit("could not create version from %q: %v", utils.PREVIOUS_BREAKING_CHANGE_VERSION, err) + } + + var versionCheckFailed bool + + if msrVoyagerVersionString == "main" { + // If the export-dir was created using the main branch, then the current version should also be the main branch. + if utils.YB_VOYAGER_VERSION != "main" { + versionCheckFailed = true + } + } else if msrVoyagerVersionString != "" { + msrVoyagerFinalVersion := msrVoyagerVersionString + if strings.Contains(msrVoyagerFinalVersion, "rc") { + msrVoyagerFinalVersion, err = utils.GetFinalReleaseVersionFromRCVersion(msrVoyagerFinalVersion) + if err != nil { + utils.ErrExit("could not get final release version from rc version %q: %v", msrVoyagerFinalVersion, err) + } + } + + msrVoyagerVersion, err := version.NewVersion(msrVoyagerFinalVersion) + if err != nil { + utils.ErrExit("could not create version from %q: %v", msrVoyagerFinalVersion, err) + } + + if msrVoyagerVersion.LessThan(previousBreakingChangeVersion) { + versionCheckFailed = true + } + } + + if versionCheckFailed { + userFacingMsg := fmt.Sprintf("\nThe export-dir %q was created using voyager version %q. "+ + "However, the current version %q requires the export-dir to be created using version %q or later. "+ + "Either use a compatible version to continue the migration or start afresh with a new export-dir. ", + migrationExportDir, msrVoyagerVersionString, utils.YB_VOYAGER_VERSION, utils.PREVIOUS_BREAKING_CHANGE_VERSION) + if msrVoyagerVersionString == "" { //In case the export dir is already started from older version that will not have VoyagerVersion field in MSR + userFacingMsg = fmt.Sprintf("\nThe export-dir %q was created using older version. "+ + "However, the current version %q requires the export-dir to be created using version %q or later. "+ + "Either use a compatible version to continue the migration or start afresh with a new export-dir. ", + migrationExportDir, utils.YB_VOYAGER_VERSION, utils.PREVIOUS_BREAKING_CHANGE_VERSION) } utils.ErrExit(userFacingMsg) } - return metaDBInstance } func initAssessmentDB() { diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index 0d105415c2..1016a51d66 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -698,3 +698,23 @@ func GetFreePort() (int, error) { addr := listener.Addr().(*net.TCPAddr) return addr.Port, nil } + +func GetFinalReleaseVersionFromRCVersion(msrVoyagerFinalVersion string) (string, error) { + // RC version will be like 0rc1.1.8.6 + // We need to extract 1.8.6 from it + // Compring with this 1.8.6 should be enough to check if the version is compatible with the current version + // Split the string at "rc" to isolate the part after it + parts := strings.Split(msrVoyagerFinalVersion, "rc") + if len(parts) > 1 { + // Further split the remaining part by '.' and remove the first segment + versionParts := strings.Split(parts[1], ".") + if len(versionParts) > 1 { + msrVoyagerFinalVersion = strings.Join(versionParts[1:], ".") // Join the parts after the first one + } else { + return "", fmt.Errorf("unexpected version format %q", msrVoyagerFinalVersion) + } + } else { + return "", fmt.Errorf("unexpected version format %q", msrVoyagerFinalVersion) + } + return msrVoyagerFinalVersion, nil +} diff --git a/yb-voyager/src/utils/version.go b/yb-voyager/src/utils/version.go index 365e5d715f..d751133c1d 100644 --- a/yb-voyager/src/utils/version.go +++ b/yb-voyager/src/utils/version.go @@ -19,6 +19,9 @@ const ( // This constant must be updated on every release. YB_VOYAGER_VERSION = "main" + // This constant must be updated after every breaking change. + PREVIOUS_BREAKING_CHANGE_VERSION = "1.8.5" + // @Refer: https://icinga.com/blog/2022/05/25/embedding-git-commit-information-in-go-binaries/ GIT_COMMIT_HASH = "$Format:%H$" ) From 2127d22599890f6dcb90eb22ef6402ab51424351 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Sat, 30 Nov 2024 11:03:10 +0530 Subject: [PATCH 024/105] Github actions workflow optimisations (#1963) * Divided pg tests in gh actions into 3 groups: offline, live_basic, live_advanced - this will help to reduce the overall time taken for the GHA to complete - Improvement: total time 57min to 26min i.e. 2x improvement * Replaced always() with !cancelled() condition from if expressions of all tests * Using yugabyted cmd to start ybdb cluster in docker container - remove old way of creating using /yb-master and /yb-tserver commands --- .github/workflows/misc-migtests.yml | 15 +++-- .github/workflows/mysql-migtests.yml | 37 ++++++----- .github/workflows/pg-9-migtests.yml | 18 ++--- .github/workflows/pg-migtests.yml | 95 ++++++++++++++------------- migtests/setup/yb-docker-compose.yaml | 46 ------------- 5 files changed, 85 insertions(+), 126 deletions(-) delete mode 100644 migtests/setup/yb-docker-compose.yaml diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 4a36f89aef..c56e4a00d2 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -71,11 +71,10 @@ jobs: ./migtests/scripts/postgresql/create_pg_user - name: "TEST: Assessment Report Test" - if: always() run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test - name: "TEST: analyze-schema" - if: always() + if: ${{ !cancelled() }} run: migtests/tests/analyze-schema/run-analyze-schema-test - name: Run import data file tests on different YugabyteDB versions @@ -85,15 +84,17 @@ jobs: GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.RAHULB_S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }} - if: always() + if: ${{ !cancelled() }} run: | versions=("2.20.5.0-b72" "2.21.1.0-b271" "2024.1.1.0-b137") for version in "${versions[@]}"; do echo "Running tests on version $version" echo "Start YugabyteDB cluster" - docker pull yugabytedb/yugabyte:$version - VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb-$version \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:$version \ + bin/yugabyted start --background=false --ui=false sleep 20 echo "Test YugabyteDB connection" @@ -114,8 +115,8 @@ jobs: migtests/tests/import-file/run-import-file-test echo "Stop the cluster before the next iteration" - VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml down --volumes - docker network prune -f + docker stop yugabytedb-$version + docker remove yugabytedb-$version done shell: bash diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 7872bea0f5..d6b434a57b 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -30,6 +30,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: Install python3 and psycopg2 run: | sudo apt install -y python3 @@ -59,8 +60,10 @@ jobs: - name: Start YugabyteDB cluster run: | - docker pull yugabytedb/yugabyte:${{ matrix.version }} - VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false sleep 20 - name: Test YugabyteDB connection @@ -77,59 +80,59 @@ jobs: psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - name: "TEST: mysql-table-list-flags-test (table-list and exclude-table-list)" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/table-list-flags-tests - name: "TEST: mysql-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/table-list-flags-tests env-file-path-flags.sh - name: "TEST: mysql-sakila" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/sakila - name: "TEST: mysql-datatypes" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/datatypes - name: "TEST: mysql-constraints" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/constraints - name: "TEST: mysql-case-indexes" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/indexes - name: "TEST: mysql-functions" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/functions - name: "TEST: mysql-case-sequences" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/sequences - name: "TEST: mysql-triggers-procedures" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/triggers-procedures - name: "TEST: mysql-case-views" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/views - name: "TEST: mysql-partitions" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/partitions - name: "TEST: mysql-sample-chinook" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/chinook - name: "TEST: mysql-misc-tests" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/misc-tests - name: "TEST: mysql-case-sensitivity-reserved-words" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/case-sensitivity-reserved-words # Placeholder for now so that a basic test can run @@ -138,5 +141,5 @@ jobs: mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';' - name: "TEST: mysql-live-migration-test" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/live-migration-run-test.sh mysql/basic-live-test diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index 622ba4134a..458804c934 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -45,6 +45,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: "Enable postgres with wal_level as logical" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" @@ -79,8 +80,10 @@ jobs: - name: Start YugabyteDB cluster run: | - docker pull yugabytedb/yugabyte:${{ matrix.version }} - VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false sleep 20 - name: Test YugabyteDB connection @@ -98,16 +101,13 @@ jobs: psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - name: "TEST: pg-case-sensitivity-single-table" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - name: "TEST: pg-datatypes" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh pg/datatypes - name: "TEST: pg-constraints" - if: always() - run: migtests/scripts/run-test.sh pg/constraints - - - + if: ${{ !cancelled() }} + run: migtests/scripts/run-test.sh pg/constraints \ No newline at end of file diff --git a/.github/workflows/pg-migtests.yml b/.github/workflows/pg-migtests.yml index 6f5af1d821..cb2457f128 100644 --- a/.github/workflows/pg-migtests.yml +++ b/.github/workflows/pg-migtests.yml @@ -12,6 +12,11 @@ jobs: matrix: version: [2.21.1.0-b271, 2024.1.1.0-b137, 2.20.5.0-b72] BETA_FAST_DATA_EXPORT: [0, 1] + test_group: + - offline + - live_basic + - live_advanced + env: BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} runs-on: ubuntu-22.04 @@ -38,6 +43,7 @@ jobs: distribution: "temurin" java-version: "17" check-latest: true + - name: Cache local Maven repository uses: actions/cache@v3 with: @@ -45,12 +51,13 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: "Enable postgres with wal_level as logical" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" docker restart ${{ job.services.postgres.id }} sleep 10 - # if: matrix.BETA_FAST_DATA_EXPORT == 1 + - name: Install python3 and psycopg2 run: | sudo apt install -y python3 @@ -79,8 +86,10 @@ jobs: - name: Start YugabyteDB cluster run: | - docker pull yugabytedb/yugabyte:${{ matrix.version }} - VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false sleep 20 - name: Test YugabyteDB connection @@ -98,120 +107,112 @@ jobs: psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/table-list-flags-tests - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh - name: "TEST: pg-case-sensitivity-single-table" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - name: "TEST: pg-dvdrental" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/dvdrental - name: "TEST: pg-datatypes" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/datatypes - name: "TEST: pg-constraints" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/constraints - name: "TEST: pg-sequences" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/sequences - name: "TEST: pg-indexes" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/indexes - name: "TEST: pg-partitions" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/partitions - name: "TEST: pg-partitions with (table-list)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions # Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529 # Fixed in 2.17.1.0-b368 - name: "TEST: pg-partitions-with-indexes" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/partitions-with-indexes - name: "TEST: pg-views-and-rules" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/views-and-rules - name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/misc-objects-1 - name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/misc-objects-2 - name: "TEST: pg-dependent-ddls" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/dependent-ddls - name: "TEST: pg-multiple-schemas" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/multiple-schemas - - name: "Set up gcp environment" - env: - GCS_CLIENT_ID: ${{ secrets.PGUPTA_GCS_CLIENT_ID }} - GCS_CLIENT_SECRET: ${{ secrets.PGUPTA_GCS_CLIENT_SECRET }} - GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }} - if: always() - run: migtests/scripts/gcs/create_gcs_credentials_file - - name: "TEST: pg-codependent-schemas" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/codependent-schemas - name: "TEST: pg-sample-schema-emp" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/sample-employee - name: "TEST: pg-hasura-ecommerce" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/hasura-ecommerce + - name: "TEST: pg-case-sensitivity-reserved-words-offline" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words + - name: "TEST: pg-basic-non-public-live-migration-test" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test - - - name: "TEST: pg-unique-key-conflicts-test" - if: always() - run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test - - # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables. - - name: "TEST: pg-live-migration-multiple-schemas" - if: always() - run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas - name: "TEST: pg-basic-public-fall-forward-test" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test # - name: "TEST: pg-basic-non-public-fall-back-test" - # if: always() # run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test - name: "TEST: pg-datatypes-fall-back-test" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes + # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables. + - name: "TEST: pg-live-migration-multiple-schemas" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas + + - name: "TEST: pg-unique-key-conflicts-test" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test + - name: "TEST: pg-live-migration-partitions-fall-forward" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions - - name: "TEST: pg-case-sensitivity-reserved-words-offline" - if: always() - run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words diff --git a/migtests/setup/yb-docker-compose.yaml b/migtests/setup/yb-docker-compose.yaml deleted file mode 100644 index e8c43262cd..0000000000 --- a/migtests/setup/yb-docker-compose.yaml +++ /dev/null @@ -1,46 +0,0 @@ -version: '2.1' - -volumes: - yb-master-data-1: - yb-tserver-data-1: - -services: - yb-master: - image: yugabytedb/yugabyte:${VERSION} - container_name: yb-master-n1 - volumes: - - yb-master-data-1:/mnt/master - command: [ "/home/yugabyte/bin/yb-master", - "--fs_data_dirs=/mnt/master", - "--master_addresses=yb-master-n1:7100", - "--rpc_bind_addresses=yb-master-n1:7100", - "--replication_factor=1"] - ports: - - "7000:7000" - - "7100:7100" - environment: - SERVICE_7000_NAME: yb-master - - yb-tserver: - image: yugabytedb/yugabyte:${VERSION} - container_name: yb-tserver-n1 - volumes: - - yb-tserver-data-1:/mnt/tserver - command: [ "/home/yugabyte/bin/yb-tserver", - "--fs_data_dirs=/mnt/tserver", - "--start_pgsql_proxy", - "--rpc_bind_addresses=yb-tserver-n1:9100", - "--tserver_master_addrs=yb-master-n1:7100"] - ports: - - "9042:9042" - - "5433:5433" - - "9000:9000" - - "9100:9100" - environment: - SERVICE_5433_NAME: ysql - SERVICE_9042_NAME: ycql - SERVICE_6379_NAME: yedis - SERVICE_9000_NAME: yb-tserver - depends_on: - - yb-master - From ed4561efe6567290d4a6082b38ff6ca2b1ed6fd3 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 2 Dec 2024 12:08:44 +0530 Subject: [PATCH 025/105] [DB-13678] Assess Migration command guardrails check for pgss extension properly loaded or not in source db (#2018) - doing SELECT query on pg_stat_statements view and comparing the error message --- yb-voyager/src/srcdb/postgres.go | 42 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 839871df5b..9f0c328e30 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -1166,10 +1166,10 @@ const ( JOIN pg_namespace n ON e.extnamespace = n.oid WHERE e.extname = 'pg_stat_statements'` - querySharedPreloadLibraries = `SELECT current_setting('shared_preload_libraries')` - queryHasReadStatsPermission = ` SELECT pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')` + + SHARED_PRELOAD_LIBRARY_ERROR = "pg_stat_statements must be loaded via shared_preload_libraries" ) // checkPgStatStatementsSetup checks if pg_stat_statements is properly installed and if the user has the necessary read permissions. @@ -1183,33 +1183,24 @@ func (pg *PostgreSQL) checkPgStatStatementsSetup() (string, error) { var pgssExtSchema string err := pg.db.QueryRow(queryPgStatStatementsSchema).Scan(&pgssExtSchema) if err != nil && err != sql.ErrNoRows { - return "", fmt.Errorf("failed to fetch the schema of pg_stat_statement available in: %w", err) + if err == sql.ErrNoRows { + return "pg_stat_statements extension is not installed on source DB, required for detecting Unsupported Query Constructs", nil + } + return "", fmt.Errorf("failed to fetch the schema of pg_stat_statements available in: %w", err) } + if pgssExtSchema == "" { return "pg_stat_statements extension is not installed on source DB, required for detecting Unsupported Query Constructs", nil } else { schemaList := lo.Union(pg.getTrimmedSchemaList(), []string{"public"}) + log.Infof("comparing schema list %v against pgss extension schema '%s'", schemaList, pgssExtSchema) if !slices.Contains(schemaList, pgssExtSchema) { return fmt.Sprintf("pg_stat_statements extension schema %q is not in the schema list (%s), required for detecting Unsupported Query Constructs", pgssExtSchema, strings.Join(schemaList, ", ")), nil } } - // TODO: finalise the approach for detecting shared_preload_libraries - // 2. check if its properly installed/loaded - // To access "shared_preload_libraries" must be superuser or a member of pg_read_all_settings - // so trying a best effort here, if accessible then check otherwise it will fail during the gather metadata - // var sharedPreloadLibraries string - // err = pg.db.QueryRow(querySharedPreloadLibraries).Scan(&sharedPreloadLibraries) - // if err != nil { - // log.Warnf("failed to check if pg_stat_statements extension is properly loaded on source DB: %v", err) - // } else { - // if !slices.Contains(strings.Split(sharedPreloadLibraries, ","), PG_STAT_STATEMENTS) { - // return "pg_stat_statements is not loaded via shared_preload_libraries, required for detecting Unsupported Query Constructs", nil - // } - // } - - // 3. User has permission to read from pg_stat_statements table + // 2. User has permission to read from pg_stat_statements table var hasReadAllStats bool err = pg.db.QueryRow(queryHasReadStatsPermission).Scan(&hasReadAllStats) if err != nil { @@ -1217,7 +1208,20 @@ func (pg *PostgreSQL) checkPgStatStatementsSetup() (string, error) { } if !hasReadAllStats { - return "User doesn't have permissions to read pg_stat_statements view, unsupported query constructs won't be detected/reported", nil + return "User doesn't have permissions to read pg_stat_statements view, required for detecting Unsupported Query Constructs", nil + } + + // To access "shared_preload_libraries" must be superuser or a member of pg_read_all_settings + // so instead of getting current_settings(), executing SELECT query on pg_stat_statements view + // 3. check if its properly installed/loaded without any extra permissions + queryCheckPgssLoaded := fmt.Sprintf("SELECT 1 from %s.pg_stat_statements LIMIT 1", pgssExtSchema) + log.Infof("query to check pgss is properly loaded - [%s]", queryCheckPgssLoaded) + _, err = pg.db.Exec(queryCheckPgssLoaded) + if err != nil { + if strings.Contains(err.Error(), SHARED_PRELOAD_LIBRARY_ERROR) { + return "pg_stat_statements is not loaded via shared_preload_libraries, required for detecting Unsupported Query Constructs", nil + } + return "", fmt.Errorf("failed to check pg_stat_statements is loaded via shared_preload_libraries: %w", err) } return "", nil From ee4cc053a8e4e561684522b769aed46fd579ffe5 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 2 Dec 2024 14:33:53 +0530 Subject: [PATCH 026/105] Filter Issues by MinimumVersionFixedIn (#1971) - New flag: target-db-version flag in analyze-schema/assess-migration user has to pass in the full version (A.B.C.D) only versions from a list of supported series can be passed. This list will be all the YB series that are not EOL. (Although officially we document that voyager only supports the two latest stable releases and one latest preview, we would NOT like to fail here if user passes 2.18, for example.) If user passes a version from a series that is not supported by voyager (say 2026.1.x.x), voyager will print the error, prompt the user if they want to continue with default (latest stable). Defaults to latest stable version (2024.1.3.0, for example) - Every Issue that we report will have the following details For every series (from the supported list), if it is fixed in any version of the series, we store that information in "minVersionsFixedIn" which is a map[series]version 2.20: fixed in 2.20.8.0 2024.1: fixed in 2024.1.1.0 2.23: fixed in 2.23.5.1 2024.2: fixed in 2024.2.1.0 - Checking if issue is fixed in target-db-version (specified by user) We compare target-db-version with Issue.minVersionsFixedIn[target-db-version.Series] - Getting list of versions that issue is fixed in (to display in reports): Option1: return all values from "minVersionsFixedIn" Option2: return values from "minVersionsFixedIn" whose series >= target-db-version.Series. Caveat: We will not be able to compare between stable and preview releases (no way to determine if 2.23 > 2024.1 or not). Therefore for target-db-version=2024.2.1.1(stable), we will omit the older stable series (2024.1 series) but include all preview series - (2.21, 2.23). --- yb-voyager/cmd/analyzeSchema.go | 29 ++-- yb-voyager/cmd/assessMigrationCommand.go | 38 ++++- yb-voyager/cmd/common.go | 9 +- yb-voyager/src/issue/issue.go | 25 ++- yb-voyager/src/issue/issue_test.go | 124 ++++++++++++++ yb-voyager/src/queryissue/queryissue.go | 52 +++++- yb-voyager/src/version/yb_version.go | 159 ------------------ yb-voyager/src/ybversion/constants.go | 40 +++++ yb-voyager/src/ybversion/yb_version.go | 115 +++++++++++++ .../{version => ybversion}/yb_version_test.go | 91 +++++----- 10 files changed, 450 insertions(+), 232 deletions(-) create mode 100644 yb-voyager/src/issue/issue_test.go delete mode 100644 yb-voyager/src/version/yb_version.go create mode 100644 yb-voyager/src/ybversion/constants.go create mode 100644 yb-voyager/src/ybversion/yb_version.go rename yb-voyager/src/{version => ybversion}/yb_version_test.go (54%) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 50af2a43d6..976e905ba3 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -41,6 +41,7 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) type summaryInfo struct { @@ -755,14 +756,14 @@ func reportUnsupportedConstraintsOnComplexDatatypesInCreate(createTableNode *pg_ for _, column := range columns { if column.GetColumnDef() != nil { /* - e.g. create table unique_def_test(id int, d daterange UNIQUE, c1 int); - create_stmt:{relation:{relname:"unique_def_test" inh:true relpersistence:"p" location:15}... - table_elts:{column_def:{colname:"d" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} - typemod:-1 location:34} is_local:true constraints:{constraint:{contype:CONSTR_UNIQUE location:38}} .... - - here checking the case where this clause is in column definition so iterating over each column_def and in that - constraint type is UNIQUE/ PK reporting that - supported. + e.g. create table unique_def_test(id int, d daterange UNIQUE, c1 int); + create_stmt:{relation:{relname:"unique_def_test" inh:true relpersistence:"p" location:15}... + table_elts:{column_def:{colname:"d" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} + typemod:-1 location:34} is_local:true constraints:{constraint:{contype:CONSTR_UNIQUE location:38}} .... + + here checking the case where this clause is in column definition so iterating over each column_def and in that + constraint type is UNIQUE/ PK reporting that + supported. */ colName := column.GetColumnDef().GetColname() typeName, ok := unsupportedColumnsForTable[colName] @@ -780,7 +781,7 @@ func reportUnsupportedConstraintsOnComplexDatatypesInCreate(createTableNode *pg_ type_name:{.... names:{string:{sval:"int4"}} typemod:-1 location:108} is_local:true location:105}} table_elts:{constraint:{contype:CONSTR_UNIQUE deferrable:true initdeferred:true location:113 keys:{string:{sval:"id"}}}} .. - here checking the case where this UK/ PK is at the end of column definition as a separate constraint + here checking the case where this UK/ PK is at the end of column definition as a separate constraint */ keys := column.GetConstraint().GetKeys() columns := []string{} @@ -1700,7 +1701,7 @@ func checker(sqlInfoArr []sqlInfo, fpath string, objType string) { func checkPlPgSQLStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { for _, sqlInfoStmt := range sqlInfoArr { - issues, err := parserIssueDetector.GetAllIssues(sqlInfoStmt.formattedStmt) + issues, err := parserIssueDetector.GetAllIssues(sqlInfoStmt.formattedStmt, targetDbVersion) if err != nil { log.Infof("error in getting the issues-%s: %v", sqlInfoStmt.formattedStmt, err) continue @@ -2316,6 +2317,10 @@ var analyzeSchemaCmd = &cobra.Command{ PreRun: func(cmd *cobra.Command, args []string) { validOutputFormats := []string{"html", "json", "txt", "xml"} validateReportOutputFormat(validOutputFormats, analyzeSchemaReportFormat) + err := validateAndSetTargetDbVersionFlag() + if err != nil { + utils.ErrExit("%v", err) + } }, Run: func(cmd *cobra.Command, args []string) { @@ -2328,6 +2333,10 @@ func init() { registerCommonGlobalFlags(analyzeSchemaCmd) analyzeSchemaCmd.PersistentFlags().StringVar(&analyzeSchemaReportFormat, "output-format", "", "format in which report can be generated: ('html', 'txt', 'json', 'xml'). If not provided, reports will be generated in both 'json' and 'html' formats by default.") + + analyzeSchemaCmd.Flags().StringVar(&targetDbVersionStrFlag, "target-db-version", "", + fmt.Sprintf("Target YugabyteDB version to analyze schema for. Defaults to latest stable version (%s)", ybversion.LatestStable.String())) + analyzeSchemaCmd.Flags().MarkHidden("target-db-version") } func validateReportOutputFormat(validOutputFormats []string, format string) { diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f235d3b2a0..754369b027 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -21,6 +21,7 @@ import ( _ "embed" "encoding/csv" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -43,6 +44,7 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/queryissue" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) var ( @@ -54,6 +56,7 @@ var ( assessMigrationSupportedDBTypes = []string{POSTGRESQL, ORACLE} referenceOrTablePartitionPresent = false ) + var sourceConnectionFlags = []string{ "source-db-host", "source-db-password", @@ -80,6 +83,10 @@ var assessMigrationCmd = &cobra.Command{ validatePortRange() validateSSLMode() validateOracleParams() + err := validateAndSetTargetDbVersionFlag() + if err != nil { + utils.ErrExit("%v", err) + } if cmd.Flags().Changed("assessment-metadata-dir") { validateAssessmentMetadataDirFlag() for _, f := range sourceConnectionFlags { @@ -284,6 +291,10 @@ func init() { "Interval (in seconds) at which voyager will gather IOPS metadata from source database for the given schema(s). (only valid for PostgreSQL)") BoolVar(assessMigrationCmd.Flags(), &source.RunGuardrailsChecks, "run-guardrails-checks", true, "run guardrails checks before assess migration. (only valid for PostgreSQL)") + + assessMigrationCmd.Flags().StringVar(&targetDbVersionStrFlag, "target-db-version", "", + fmt.Sprintf("Target YugabyteDB version to assess migration for. Defaults to latest stable version (%s)", ybversion.LatestStable.String())) + assessMigrationCmd.Flags().MarkHidden("target-db-version") } func assessMigration() (err error) { @@ -1102,8 +1113,7 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error for i := 0; i < len(executedQueries); i++ { query := executedQueries[i] log.Debugf("fetching unsupported query constructs for query - [%s]", query) - - issues, err := parserIssueDetector.GetDMLIssues(query) + issues, err := parserIssueDetector.GetDMLIssues(query, targetDbVersion) if err != nil { log.Errorf("failed while trying to fetch query issues in query - [%s]: %v", query, err) @@ -1400,3 +1410,27 @@ func validateAssessmentMetadataDirFlag() { } } } + +func validateAndSetTargetDbVersionFlag() error { + if targetDbVersionStrFlag == "" { + targetDbVersion = ybversion.LatestStable + utils.PrintAndLog("Defaulting to latest stable YugabyteDB version: %s", targetDbVersion) + return nil + } + var err error + targetDbVersion, err = ybversion.NewYBVersion(targetDbVersionStrFlag) + + if err == nil || !errors.Is(err, ybversion.ErrUnsupportedSeries) { + return err + } + + // error is ErrUnsupportedSeries + utils.PrintAndLog("%v", err) + if utils.AskPrompt("Do you want to continue with the latest stable YugabyteDB version:", ybversion.LatestStable.String()) { + targetDbVersion = ybversion.LatestStable + return nil + } else { + utils.ErrExit("Aborting..") + return nil + } +} diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index df1cb4f25e..0418797e2e 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -59,12 +59,15 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/jsonfile" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) var ( - metaDB *metadb.MetaDB - PARENT_COMMAND_USAGE = "Parent command. Refer to the sub-commands for usage help." - startTime time.Time + metaDB *metadb.MetaDB + PARENT_COMMAND_USAGE = "Parent command. Refer to the sub-commands for usage help." + startTime time.Time + targetDbVersionStrFlag string + targetDbVersion *ybversion.YBVersion ) func PrintElapsedDuration() { diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index 2e43ebbada..a7baef18a8 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -16,13 +16,26 @@ limitations under the License. package issue +import ( + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" +) + type Issue struct { - Type string // (advisory_locks, index_not_supported, etc) - TypeName string // for display - TypeDescription string - Suggestion string - GH string - DocsLink string + Type string // (advisory_locks, index_not_supported, etc) + TypeName string // for display + TypeDescription string + Suggestion string + GH string + DocsLink string + MinimumVersionsFixedIn map[string]*ybversion.YBVersion // key: series (2024.1, 2.21, etc) +} + +func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { + minVersionFixedInSeries, ok := i.MinimumVersionsFixedIn[v.Series()] + if !ok { + return false, nil + } + return v.GreaterThanOrEqual(minVersionFixedInSeries), nil } type IssueInstance struct { diff --git a/yb-voyager/src/issue/issue_test.go b/yb-voyager/src/issue/issue_test.go new file mode 100644 index 0000000000..97042daf2b --- /dev/null +++ b/yb-voyager/src/issue/issue_test.go @@ -0,0 +1,124 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package issue + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" +) + +func TestIssueFixedInStable(t *testing.T) { + fixedVersion, err := ybversion.NewYBVersion("2024.1.1.0") + assert.NoError(t, err) + issue := Issue{ + Type: ADVISORY_LOCKS, + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ + ybversion.SERIES_2024_1: fixedVersion, + }, + } + + versionsToCheck := map[string]bool{ + "2024.1.1.0": true, + "2024.1.1.1": true, + "2024.1.0.0": false, + } + for v, expected := range versionsToCheck { + ybVersion, err := ybversion.NewYBVersion(v) + assert.NoError(t, err) + + fixed, err := issue.IsFixedIn(ybVersion) + assert.NoError(t, err) + assert.Equalf(t, expected, fixed, "comparing ybv %s to fixed %s", ybVersion, fixedVersion) + } +} + +func TestIssueFixedInPreview(t *testing.T) { + fixedVersion, err := ybversion.NewYBVersion("2.21.4.5") + assert.NoError(t, err) + issue := Issue{ + Type: ADVISORY_LOCKS, + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ + ybversion.SERIES_2_21: fixedVersion, + }, + } + + versionsToCheck := map[string]bool{ + "2.21.4.5": true, + "2.21.5.5": true, + "2.21.4.1": false, + } + for v, expected := range versionsToCheck { + ybVersion, err := ybversion.NewYBVersion(v) + assert.NoError(t, err) + + fixed, err := issue.IsFixedIn(ybVersion) + assert.NoError(t, err) + assert.Equalf(t, expected, fixed, "comparing ybv %s to fixed %s", ybVersion, fixedVersion) + } +} + +func TestIssueFixedInStableOld(t *testing.T) { + fixedVersionStableOld, err := ybversion.NewYBVersion("2.20.7.1") + assert.NoError(t, err) + fixedVersionStable, err := ybversion.NewYBVersion("2024.1.1.1") + assert.NoError(t, err) + + issue := Issue{ + Type: ADVISORY_LOCKS, + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ + ybversion.SERIES_2024_1: fixedVersionStable, + ybversion.SERIES_2_20: fixedVersionStableOld, + }, + } + + versionsToCheck := map[string]bool{ + "2.20.0.0": false, + "2.20.7.0": false, + "2.20.7.1": true, + "2024.1.1.1": true, + "2024.1.1.2": true, + "2024.1.1.0": false, + } + for v, expected := range versionsToCheck { + ybVersion, err := ybversion.NewYBVersion(v) + assert.NoError(t, err) + + fixed, err := issue.IsFixedIn(ybVersion) + assert.NoError(t, err) + assert.Equalf(t, expected, fixed, "comparing ybv %s to fixed [%s, %s]", ybVersion, fixedVersionStableOld, fixedVersionStable) + } +} + +func TestIssueFixedFalseWhenMinimumNotSpecified(t *testing.T) { + issue := Issue{ + Type: ADVISORY_LOCKS, + } + + versionsToCheck := []string{"2024.1.0.0", "2.20.7.4", "2.21.1.1"} + + for _, v := range versionsToCheck { + ybVersion, err := ybversion.NewYBVersion(v) + assert.NoError(t, err) + + fixed, err := issue.IsFixedIn(ybVersion) + assert.NoError(t, err) + // If the minimum fixed version is not specified, the issue is not fixed in any version. + assert.Falsef(t, fixed, "comparing ybv %s to fixed should be false", ybVersion) + } +} diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index f63ee0e476..9772e3a160 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -25,6 +25,7 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) type ParserIssueDetector struct { @@ -36,11 +37,35 @@ func NewParserIssueDetector() *ParserIssueDetector { return &ParserIssueDetector{} } -func (p *ParserIssueDetector) GetAllIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []issue.IssueInstance, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { + var filteredIssues []issue.IssueInstance + for _, i := range issues { + fixed, err := i.IsFixedIn(targetDbVersion) + if err != nil { + return nil, fmt.Errorf("checking if issue %v is supported: %w", i, err) + } + if !fixed { + filteredIssues = append(filteredIssues, i) + } + } + return filteredIssues, nil +} + +func (p *ParserIssueDetector) GetAllIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { + issues, err := p.getAllIssues(query) + if err != nil { + return issues, err + } + + return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) +} + +func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) } + if queryparser.IsPLPGSQLObject(parseTree) { objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) plpgsqlQueries, err := queryparser.GetAllPLPGSQLStatements(query) @@ -49,7 +74,7 @@ func (p *ParserIssueDetector) GetAllIssues(query string) ([]issue.IssueInstance, } var issues []issue.IssueInstance for _, plpgsqlQuery := range plpgsqlQueries { - issuesInQuery, err := p.GetAllIssues(plpgsqlQuery) + issuesInQuery, err := p.getAllIssues(plpgsqlQuery) if err != nil { //there can be plpgsql expr queries no parseable via parser e.g. "withdrawal > balance" log.Errorf("error getting issues in query-%s: %v", query, err) @@ -57,6 +82,7 @@ func (p *ParserIssueDetector) GetAllIssues(query string) ([]issue.IssueInstance, } issues = append(issues, issuesInQuery...) } + return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object //e.g. replacing the DML_QUERY and "" to FUNCTION and @@ -72,10 +98,11 @@ func (p *ParserIssueDetector) GetAllIssues(query string) ([]issue.IssueInstance, if err != nil { return nil, fmt.Errorf("error deparsing a select stmt: %v", err) } - issues, err := p.GetAllIssues(selectStmtQuery) + issues, err := p.getAllIssues(selectStmtQuery) if err != nil { return nil, err } + return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object //e.g. replacing the DML_QUERY and "" to FUNCTION and @@ -85,12 +112,25 @@ func (p *ParserIssueDetector) GetAllIssues(query string) ([]issue.IssueInstance, }), nil } - return p.GetDMLIssues(query) + + issues, err := p.getDMLIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting DML issues: %w", err) + } + return issues, nil } -//TODO: in future when we will DDL issues detection here we need `GetDDLIssues` +func (p *ParserIssueDetector) GetDMLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { + issues, err := p.getDMLIssues(query) + if err != nil { + return issues, err + } + + return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) +} -func (p *ParserIssueDetector) GetDMLIssues(query string) ([]issue.IssueInstance, error) { +// TODO: in future when we will DDL issues detection here we need `GetDDLIssues` +func (p *ParserIssueDetector) getDMLIssues(query string) ([]issue.IssueInstance, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) diff --git a/yb-voyager/src/version/yb_version.go b/yb-voyager/src/version/yb_version.go deleted file mode 100644 index e5ffd7d14a..0000000000 --- a/yb-voyager/src/version/yb_version.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright (c) YugabyteDB, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package version - -import ( - "fmt" - "strconv" - "strings" - - "github.com/hashicorp/go-version" - "github.com/samber/lo" - "golang.org/x/exp/slices" -) - -// Reference - https://docs.yugabyte.com/preview/releases/ybdb-releases/ -var supportedYBVersionStableSeriesOld = []string{"2.14", "2.18", "2.20"} -var supportedYBVersionStableSeries = []string{"2024.1"} -var supportedYBVersionPreviewSeries = []string{"2.21", "2.23"} - -var allSupportedYBVersionSeries = lo.Flatten([][]string{supportedYBVersionStableSeries, supportedYBVersionPreviewSeries, supportedYBVersionStableSeriesOld}) - -const ( - STABLE = "stable" - PREVIEW = "preview" - STABLE_OLD = "stable_old" -) - -/* -YBVersion is a wrapper around hashicorp/go-version.Version that adds some Yugabyte-specific -functionality. - 1. It only supports versions with 2, 3, or 4 segments (A.B, A.B.C, A.B.C.D). - 2. It only accepts one of supported Yugabyte version series. - 3. It provides a method to compare versions based on common prefix of segments. This is useful in cases - where the version is not fully specified (e.g. 2.20 vs 2.20.7.0). - If the user specifies 2.20, a reasonable assumption is that they would be on the latest 2.20.x.y version. - Therefore, we want only want to compare the prefix 2.20 between versions and ignore the rest. -*/ -type YBVersion struct { - *version.Version -} - -func NewYBVersion(v string) (*YBVersion, error) { - v1, err := version.NewVersion(v) - if err != nil { - return nil, err - } - - ybv := &YBVersion{v1} - origSegLen := ybv.originalSegmentsLen() - if origSegLen < 2 || origSegLen > 4 { - return nil, fmt.Errorf("invalid YB version: %s. Version should have between min 2 and max 4 segments (A.B.C.D). Version %s has ybv.originalSegmentsLen()", v, v) - } - - if !slices.Contains(allSupportedYBVersionSeries, ybv.Series()) { - return nil, fmt.Errorf("unsupported YB version series: %s. Supported YB version series = %v", ybv.Series(), allSupportedYBVersionSeries) - } - return ybv, nil -} - -// The first two segments essentially represent the release series -// as per https://docs.yugabyte.com/preview/releases/ybdb-releases/ -func (ybv *YBVersion) Series() string { - return joinIntsWith(ybv.Segments()[:2], ".") -} - -func (ybv *YBVersion) ReleaseType() string { - series := ybv.Series() - if slices.Contains(supportedYBVersionStableSeries, series) { - return STABLE - } else if slices.Contains(supportedYBVersionPreviewSeries, series) { - return PREVIEW - } else if slices.Contains(supportedYBVersionStableSeriesOld, series) { - return STABLE_OLD - } else { - panic("unknown release type for series: " + series) - } -} - -// This returns the len of the segments in the original -// input. For instance if input is 2024.1, -// go-version.Version.Segments() will return [2024, 1, 0, 0] -// original = 2024.1 -// originalSegmentsLen = 2 ([2024,1]) -func (ybv *YBVersion) originalSegmentsLen() int { - orig := ybv.Original() - segments := strings.Split(orig, ".") - return len(segments) -} - -/* -Compare the common prefix of segments between two versions. -Similar to method go-version.Version.Compare, but only compares the common prefix of segments. -This is useful in cases where the version is not fully specified (e.g. 2.20 vs 2.20.7.0). -If the user wants to compare 2.20 and 2.20.7.0, a reasonable assumption is that they would be on the latest 2.20.x.y version. -Therefore, we want only want to compare the prefix 2.20 between versions and ignore the rest. - -This returns -1, 0, or 1 if this version is smaller, equal, -or larger than the other version, respectively. -*/ -func (ybv *YBVersion) CompareCommonPrefix(other *YBVersion) (int, error) { - if ybv.Series() != other.Series() { - return 0, fmt.Errorf("cannot compare versions with different series: %s and %s", ybv.Series(), other.Series()) - } - myOriginalSegLen := ybv.originalSegmentsLen() - otherOriginalSegLen := other.originalSegmentsLen() - minSegLen := min(myOriginalSegLen, otherOriginalSegLen) - - ybvMin, err := version.NewVersion(joinIntsWith(ybv.Segments()[:minSegLen], ".")) - if err != nil { - return 0, fmt.Errorf("create version from segments: %v", ybv.Segments()[:minSegLen]) - } - otherMin, err := version.NewVersion(joinIntsWith(other.Segments()[:minSegLen], ".")) - if err != nil { - return 0, fmt.Errorf("create version from segments: %v", other.Segments()[:minSegLen]) - } - return ybvMin.Compare(otherMin), nil -} - -func (ybv *YBVersion) CommonPrefixGreaterThanOrEqual(other *YBVersion) (bool, error) { - res, err := ybv.CompareCommonPrefix(other) - if err != nil { - return false, err - } - return res >= 0, nil -} - -func (ybv *YBVersion) CommonPrefixLessThan(other *YBVersion) (bool, error) { - res, err := ybv.CompareCommonPrefix(other) - if err != nil { - return false, err - } - return res < 0, nil -} - -func (ybv *YBVersion) String() string { - return ybv.Original() -} - -func joinIntsWith(ints []int, delimiter string) string { - strs := make([]string, len(ints)) - for i, v := range ints { - strs[i] = strconv.Itoa(v) - } - return strings.Join(strs, delimiter) -} diff --git a/yb-voyager/src/ybversion/constants.go b/yb-voyager/src/ybversion/constants.go new file mode 100644 index 0000000000..1e6f4afecc --- /dev/null +++ b/yb-voyager/src/ybversion/constants.go @@ -0,0 +1,40 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package ybversion + +import "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + +const ( + SERIES_2_14 = "2.14" + SERIES_2_18 = "2.18" + SERIES_2_20 = "2.20" + SERIES_2024_1 = "2024.1" + SERIES_2_21 = "2.21" + SERIES_2_23 = "2.23" +) + +var LatestStable *YBVersion + +var V2024_1_3_1 *YBVersion + +func init() { + var err error + V2024_1_3_1, err = NewYBVersion("2024.1.3.1") + if err != nil { + utils.ErrExit("could not create version 2024.1") + } + LatestStable = V2024_1_3_1 +} diff --git a/yb-voyager/src/ybversion/yb_version.go b/yb-voyager/src/ybversion/yb_version.go new file mode 100644 index 0000000000..dc4aab5437 --- /dev/null +++ b/yb-voyager/src/ybversion/yb_version.go @@ -0,0 +1,115 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ybversion + +import ( + "fmt" + "strconv" + "strings" + + "github.com/hashicorp/go-version" + "github.com/samber/lo" + "golang.org/x/exp/slices" +) + +// Reference - https://docs.yugabyte.com/preview/releases/ybdb-releases/ +var supportedYBVersionStableSeriesOld = []string{SERIES_2_14, SERIES_2_18, SERIES_2_20} +var supportedYBVersionStableSeries = []string{SERIES_2024_1} +var supportedYBVersionPreviewSeries = []string{SERIES_2_21, SERIES_2_23} + +var allSupportedYBVersionSeries = lo.Flatten([][]string{supportedYBVersionStableSeries, supportedYBVersionPreviewSeries, supportedYBVersionStableSeriesOld}) +var ErrUnsupportedSeries = fmt.Errorf("unsupported YB version series. Supported YB version series = %v", allSupportedYBVersionSeries) + +const ( + STABLE = "stable" + PREVIEW = "preview" + STABLE_OLD = "stable_old" +) + +/* +YBVersion is a wrapper around hashicorp/go-version.Version that adds some Yugabyte-specific +functionality. + 1. It only supports versions with 4 segments (A.B.C.D). + 2. It only accepts one of supported Yugabyte version series. +*/ +type YBVersion struct { + *version.Version +} + +func NewYBVersion(v string) (*YBVersion, error) { + v1, err := version.NewVersion(v) + if err != nil { + return nil, err + } + + ybv := &YBVersion{v1} + origSegLen := ybv.OriginalSegmentsLen() + if origSegLen != 4 { + return nil, fmt.Errorf("invalid YB version: %s. It has %d segments. Version should have exactly 4 segments (A.B.C.D).", v, origSegLen) + } + + if !slices.Contains(allSupportedYBVersionSeries, ybv.Series()) { + return nil, ErrUnsupportedSeries + } + return ybv, nil +} + +// The first two segments essentially represent the release series +// as per https://docs.yugabyte.com/preview/releases/ybdb-releases/ +func (ybv *YBVersion) Series() string { + return joinIntsWith(ybv.Segments()[:2], ".") +} + +func (ybv *YBVersion) ReleaseType() string { + series := ybv.Series() + if slices.Contains(supportedYBVersionStableSeries, series) { + return STABLE + } else if slices.Contains(supportedYBVersionPreviewSeries, series) { + return PREVIEW + } else if slices.Contains(supportedYBVersionStableSeriesOld, series) { + return STABLE_OLD + } else { + panic("unknown release type for series: " + series) + } +} + +// This returns the len of the segments in the original +// input. For instance if input is 2024.1, +// go-version.Version.Segments() will return [2024, 1, 0, 0] +// original = 2024.1 +// originalSegmentsLen = 2 ([2024,1]) +func (ybv *YBVersion) OriginalSegmentsLen() int { + orig := ybv.Original() + segments := strings.Split(orig, ".") + return len(segments) +} + +func (ybv *YBVersion) GreaterThanOrEqual(other *YBVersion) bool { + return ybv.Version.GreaterThanOrEqual(other.Version) +} + +func (ybv *YBVersion) String() string { + return ybv.Original() +} + +func joinIntsWith(ints []int, delimiter string) string { + strs := make([]string, len(ints)) + for i, v := range ints { + strs[i] = strconv.Itoa(v) + } + return strings.Join(strs, delimiter) +} diff --git a/yb-voyager/src/version/yb_version_test.go b/yb-voyager/src/ybversion/yb_version_test.go similarity index 54% rename from yb-voyager/src/version/yb_version_test.go rename to yb-voyager/src/ybversion/yb_version_test.go index 024d991ab1..dec6300480 100644 --- a/yb-voyager/src/version/yb_version_test.go +++ b/yb-voyager/src/ybversion/yb_version_test.go @@ -14,9 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package version +package ybversion import ( + "encoding/json" + "io" + "net/http" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -24,8 +28,8 @@ import ( func TestValidNewYBVersion(t *testing.T) { validVersionStrings := []string{ - "2024.1.1", - "2.20", + "2024.1.1.0", + "2.20.7.0", "2.21.2.1", } for _, v := range validVersionStrings { @@ -38,8 +42,9 @@ func TestInvalidNewYBVersion(t *testing.T) { invalidVersionStrings := []string{ "abc.def", // has to be numbers "2024.0.1-1", // has to be in supported series - "2024", // has to have at least 2 segments - "2024.1.1.1.1.1", // max 4 segments + "2024", // has to have 4 segments + "2.20.7", // has to have 4 segments + "2024.1.1.1.1.1", // exactly 4 segments } for _, v := range invalidVersionStrings { _, err := NewYBVersion(v) @@ -49,8 +54,8 @@ func TestInvalidNewYBVersion(t *testing.T) { func TestStableReleaseType(t *testing.T) { stableVersionStrings := []string{ - "2024.1.1", - "2024.1", + "2024.1.1.0", + "2024.1.0.0", "2024.1.1.1", } for _, v := range stableVersionStrings { @@ -61,8 +66,8 @@ func TestStableReleaseType(t *testing.T) { func TestPreviewReleaseType(t *testing.T) { previewVersionStrings := []string{ - "2.21.1", - "2.21", + "2.21.1.0", + "2.21.1.1", } for _, v := range previewVersionStrings { ybVersion, _ := NewYBVersion(v) @@ -72,8 +77,8 @@ func TestPreviewReleaseType(t *testing.T) { func TestStableOldReleaseType(t *testing.T) { stableOldVersionStrings := []string{ - "2.20.1", - "2.20", + "2.20.1.0", + "2.20.0.0", } for _, v := range stableOldVersionStrings { ybVersion, _ := NewYBVersion(v) @@ -81,42 +86,36 @@ func TestStableOldReleaseType(t *testing.T) { } } -func TestVersionPrefixGreaterThanOrEqual(t *testing.T) { - versionsToCompare := [][]string{ - {"2024.1.1", "2024.1.0"}, - {"2024.1", "2024.1.4.0"}, //prefix 2024.1 == 2024.1 +func TestLatestStable(t *testing.T) { + type Release struct { + Name string `json:"name"` } - for _, v := range versionsToCompare { - v1, err := NewYBVersion(v[0]) - assert.NoError(t, err) - v2, err := NewYBVersion(v[1]) - assert.NoError(t, err) - greaterThan, err := v1.CommonPrefixGreaterThanOrEqual(v2) - assert.NoError(t, err) - assert.True(t, greaterThan, "%s >= %s", v[0], v[1]) - } -} -func TestVersionPrefixLessThan(t *testing.T) { - versionsToCompare := [][]string{ - {"2024.1.0", "2024.1.2"}, - {"2.21.1", "2.21.7"}, - {"2.20.1", "2.20.7"}, - } - for _, v := range versionsToCompare { - v1, err := NewYBVersion(v[0]) - assert.NoError(t, err) - v2, err := NewYBVersion(v[1]) - assert.NoError(t, err) - lessThan, err := v1.CommonPrefixLessThan(v2) - assert.NoError(t, err) - assert.True(t, lessThan, "%s < %s", v[0], v[1]) - } -} + url := "https://api.github.com/repos/yugabyte/yugabyte-db/releases" + response, err := http.Get(url) + assert.NoErrorf(t, err, "could not access URL:%q", url) + defer response.Body.Close() + assert.Equal(t, 200, response.StatusCode) + + body, err := io.ReadAll(response.Body) + assert.NoError(t, err, "could not read contents of response %q") + var releases []Release + err = json.Unmarshal(body, &releases) + assert.NoErrorf(t, err, "could not unmarshal response %q", string(body)) + assert.NotEmpty(t, releases, "no releases found") -func TestComparingDifferentSeriesThrowsError(t *testing.T) { - v1, _ := NewYBVersion("2024.1.1") - v2, _ := NewYBVersion("2.21.1") - _, err := v1.CompareCommonPrefix(v2) - assert.Error(t, err) + for _, r := range releases { + // sample - v2.20.7.1 (Released October 16, 2024) + releaseName := r.Name + releaseName = strings.Split(releaseName, " ")[0] + if releaseName[0] == 'v' { + releaseName = releaseName[1:] + } + releaseName = strings.Trim(releaseName, " ") + rVersion, err := NewYBVersion(releaseName) + assert.NoErrorf(t, err, "could not create version %q", releaseName) + if rVersion.ReleaseType() == STABLE { + assert.True(t, LatestStable.GreaterThanOrEqual(rVersion), "%s is not greater than %s", LatestStable, rVersion) + } + } } From b6e0880bcc7c909e1dc14f779b323ca9414955c5 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:24:20 +0530 Subject: [PATCH 027/105] Modifications in cpan modules installation in the airgapped installer script (#2021) --- .../install-voyager-airgapped.sh | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/installer_scripts/install-voyager-airgapped.sh b/installer_scripts/install-voyager-airgapped.sh index 886ed06148..3a1a1c49ae 100644 --- a/installer_scripts/install-voyager-airgapped.sh +++ b/installer_scripts/install-voyager-airgapped.sh @@ -150,8 +150,12 @@ install_perl_module() { local requirement_type="$2" local required_version="$3" local package="$4" - - echo "Installing module $module_name..." + + # Check if the module is already installed and meets the version requirements + check_perl_module_version "$module_name" "$requirement_type" "$required_version" "true" + if [[ $? -eq 0 ]]; then + return + fi # Extract the package tar -xzvf "$package" 1>&2 || { echo "Error: Failed to extract $package"; exit 1; } @@ -183,35 +187,56 @@ install_perl_module() { # Return to the original directory cd .. - # Verification and version check + # Verification of the installed module + check_perl_module_version "$module_name" "$requirement_type" "$required_version" "false" + if [[ $? -ne 0 ]]; then + exit 1 + fi +} + +check_perl_module_version() { + local module_name="$1" + local requirement_type="$2" + local required_version="$3" + local check_only="$4" # If "true", suppress error messages and exit silently + + # Get installed version + local installed_version installed_version=$(perl -M"$module_name" -e 'print $'"$module_name"'::VERSION' 2> /dev/null) - + if [[ -z "$installed_version" ]]; then - echo "Error: $module_name could not be loaded or found." - exit 1 + if [[ "$check_only" != "true" ]]; then + echo "Error: $module_name could not be loaded or found." + fi + return 1 fi # Version comparison based on requirement type if [[ "$requirement_type" == "min" ]]; then # Check if installed version is at least the required version - if [[ $(echo -e "$installed_version\n$required_version" | sort -V | head -n1) != "$required_version" ]]; then + if [[ $(echo -e "$installed_version\n$required_version" | sort -V | head -n1) == "$required_version" ]]; then + return 0 + fi + if [[ "$check_only" != "true" ]]; then echo "Error: Installed version of $module_name ($installed_version) does not meet the minimum required version ($required_version)." - exit 1 fi + return 1 elif [[ "$requirement_type" == "exact" ]]; then # Check if installed version matches the required version exactly - if [[ "$installed_version" != "$required_version" ]]; then + if [[ "$installed_version" == "$required_version" ]]; then + return 0 + fi + if [[ "$check_only" != "true" ]]; then echo "Error: Installed version of $module_name ($installed_version) does not match the exact required version ($required_version)." - exit 1 fi + return 1 else echo "Error: Unknown requirement type '$requirement_type' for $module_name." exit 1 fi - - echo "" } + check_binutils_version() { min_required_version='2.25' @@ -428,9 +453,6 @@ centos_main() { echo "" echo -e "\e[33mYum packages:\e[0m" print_dependencies "${centos_yum_package_requirements[@]}" - echo "" - echo -e "\e[33mCPAN modules:\e[0m" - print_dependencies "${cpan_modules_requirements[@]}" print_steps_to_install_oic_on_centos exit 0 fi @@ -609,9 +631,6 @@ ubuntu_main() { echo "" echo -e "\e[33mApt packages:\e[0m" print_dependencies "${ubuntu_apt_package_requirements[@]}" - echo "" - echo -e "\e[33mCPAN modules:\e[0m" - print_dependencies "${cpan_modules_requirements[@]}" print_steps_to_install_oic_on_ubuntu exit 0 fi From 0f7145191a80c0ea02fb4643b9adda5059fb2e34 Mon Sep 17 00:00:00 2001 From: Hemant Bhanawat Date: Wed, 4 Dec 2024 10:21:46 +0530 Subject: [PATCH 028/105] Pull request template (#2029) Co-authored-by: hbhanawat --- .github/PULL_REQUEST_TEMPLATE | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 0000000000..2874dbc9b8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,37 @@ +### Describe the changes in this pull request + + +### Describe if there are any user-facing changes + + +### How was this pull request tested? + + +### Does your PR have changes that can cause upgrade issues? +| Component | Breaking changes? | +| :----------------------------------------------: | :-----------: | +| MetaDB | Yes/No | +| Name registry json | Yes/No | +| Data File Descriptor Json | Yes/No | +| Export Snapshot Status Json | Yes/No | +| Import Data State | Yes/No | +| Export Status Json | Yes/No | +| Data .sql files of tables | Yes/No | +| Export and import data queue | Yes/No | +| Schema Dump | Yes/No | +| AssessmentDB | Yes/No | +| Sizing DB | Yes/No | +| Migration Assessment Report Json | Yes/No | +| Callhome Json | Yes/No | +| YugabyteD Tables | Yes/No | +| TargetDB Metadata Tables | Yes/No | From d46d4fa48dd475305fce90f859695f1992338da1 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:04:52 +0530 Subject: [PATCH 029/105] Added check to ensure certain commands don't give segmentation fault errors when run before migration has been started (#2022) Before running archives changes command that following checks are done: Code firstly checks if meta.db is present or not 1. If the the first check passes then it checks whether msr.ExportDataSourceDebeziumStarted is true or not 2. If either of the two checks fail, we exit with an appropriate message. Also added the check number 1 in case of commands end migration, import data status, export data status, cutover to source, cutover to source replica, cutover to target, cutover status --- yb-voyager/cmd/archiveChangesCommand.go | 9 +++++++ yb-voyager/cmd/root.go | 33 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/yb-voyager/cmd/archiveChangesCommand.go b/yb-voyager/cmd/archiveChangesCommand.go index 945c69ee9a..c47f5f9389 100644 --- a/yb-voyager/cmd/archiveChangesCommand.go +++ b/yb-voyager/cmd/archiveChangesCommand.go @@ -55,6 +55,15 @@ func archiveChangesCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("one of the --move-to and --delete-changes-without-archiving must be set") } + // Check to ensure that export data with live migration is running + msr, err := metaDB.GetMigrationStatusRecord() + if err != nil { + utils.ErrExit("Error getting migration status record: %v", err) + } + if !msr.ExportDataSourceDebeziumStarted { + utils.ErrExit("The streaming phase of export data has not started yet. This command can only be run after the streaming phase begins.") + } + metaDB.UpdateMigrationStatusRecord(func(record *metadb.MigrationStatusRecord) { record.ArchivingEnabled = true }) diff --git a/yb-voyager/cmd/root.go b/yb-voyager/cmd/root.go index 7cca3f1b2f..67002b631d 100644 --- a/yb-voyager/cmd/root.go +++ b/yb-voyager/cmd/root.go @@ -23,6 +23,7 @@ import ( "path/filepath" "time" + "github.com/fatih/color" "github.com/google/uuid" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -63,6 +64,7 @@ Refer to docs (https://docs.yugabyte.com/preview/migrate/) for more details like PersistentPreRun: func(cmd *cobra.Command, args []string) { currentCommand = cmd.CommandPath() + if !shouldRunPersistentPreRun(cmd) { return } @@ -87,6 +89,7 @@ Refer to docs (https://docs.yugabyte.com/preview/migrate/) for more details like startTime = time.Now() log.Infof("Start time: %s\n", startTime) + // Initialize the metaDB variable only if the metaDB is already created. For example, resumption of a command. metaDB = initMetaDB(bulkAssessmentDir) if perfProfile { go startPprofServer() @@ -114,9 +117,15 @@ Refer to docs (https://docs.yugabyte.com/preview/migrate/) for more details like startTime = time.Now() log.Infof("Start time: %s\n", startTime) + if shouldRunExportDirInitialisedCheck(cmd) { + checkExportDirInitialised() + } + if callhome.SendDiagnostics { go sendCallhomePayloadAtIntervals() } + + // Initialize the metaDB variable only if the metaDB is already created. For example, resumption of a command. if metaDBIsCreated(exportDir) { metaDB = initMetaDB(exportDir) } @@ -146,6 +155,18 @@ Refer to docs (https://docs.yugabyte.com/preview/migrate/) for more details like }, } +func shouldRunExportDirInitialisedCheck(cmd *cobra.Command) bool { + return slices.Contains(exportDirInitialisedCheckNeededList, cmd.CommandPath()) +} + +func checkExportDirInitialised() { + // Check to ensure that this is not the first command in the migration process + isMetaDBPresent := metaDBIsCreated(exportDir) + if !isMetaDBPresent { + utils.ErrExit("Migration has not started yet. Run the commands in the order specified in the documentation: %s", color.BlueString("https://docs.yugabyte.com/preview/yugabyte-voyager/migrate/")) + } +} + func startPprofServer() { // Server for pprof err := http.ListenAndServe("localhost:6060", nil) @@ -161,6 +182,18 @@ func startPprofServer() { */ } +var exportDirInitialisedCheckNeededList = []string{ + "yb-voyager import data status", + "yb-voyager export data status", + "yb-voyager cutover status", + "yb-voyager get data-migration-report", + "yb-voyager archive changes", + "yb-voyager end migration", + "yb-voyager initiate cutover to source", + "yb-voyager initiate cutover to source-replica", + "yb-voyager initiate cutover to target", +} + var noLockNeededList = []string{ "yb-voyager", "yb-voyager version", From 995f770b984a33d7c9e5a6df79754ac89897e152 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 4 Dec 2024 18:07:13 +0530 Subject: [PATCH 030/105] Reporting %TYPE syntax for type names in the Function and Procedure objects (#2004) Reporting the %TYPE syntax issues in FUNCTION and PROCEDURE objects in Variable declaration, function parameters, and return type of function. Added tests in the analyze and assessment test. --- .../schema/functions/function.sql | 88 +++- .../schema/procedures/procedure.sql | 24 + .../tests/analyze-schema/expected_issues.json | 140 +++++ migtests/tests/analyze-schema/summary.json | 12 +- .../expectedAssessmentReport.json | 29 +- .../pg_assessment_report.sql | 53 ++ .../expectedAssessmentReport.json | 231 ++++++++- .../expected_schema_analysis_report.json | 486 +++++++++++++++++- yb-voyager/cmd/analyzeSchema.go | 2 + yb-voyager/src/issue/constants.go | 7 +- yb-voyager/src/issue/ddl.go | 31 ++ yb-voyager/src/queryissue/queryissue.go | 38 +- yb-voyager/src/queryparser/query_parser.go | 77 ++- .../src/queryparser/traversal_plpgsql.go | 217 +++++++- 14 files changed, 1393 insertions(+), 42 deletions(-) create mode 100644 yb-voyager/src/issue/ddl.go diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql index f85ea54d71..fc5c894f6d 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql @@ -87,4 +87,90 @@ BEGIN SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department); END; -$$; \ No newline at end of file +$$; + +CREATE FUNCTION public.get_employeee_salary(emp_id integer) RETURNS numeric + LANGUAGE plpgsql + AS $$ +DECLARE + emp_salary employees.salary%TYPE; -- Declare a variable with the same type as employees.salary +BEGIN + SELECT salary INTO emp_salary + FROM employees + WHERE employee_id = emp_id; + RETURN emp_salary; +END; +$$; + +CREATE OR REPLACE FUNCTION calculate_tax(salary_amount NUMERIC) RETURNS NUMERIC AS $$ +DECLARE + tax_rate employees.tax_rate%TYPE; -- Inherits type from employees.tax_rate column + tax_amount NUMERIC; +BEGIN + -- Assign a value to the variable + SELECT tax_rate INTO tax_rate FROM employees WHERE id = 1; + + -- Use the variable in a calculation + tax_amount := salary_amount * tax_rate; + RETURN tax_amount; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION log_salary_change() RETURNS TRIGGER AS $$ +DECLARE + old_salary employees.salary%TYPE; -- Matches the type of the salary column + new_salary employees.salary%TYPE; +BEGIN + old_salary := OLD.salary; + new_salary := NEW.salary; + + IF new_salary <> old_salary THEN + INSERT INTO salary_log(employee_id, old_salary, new_salary, changed_at) + VALUES (NEW.id, old_salary, new_salary, now()); + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER salary_update_trigger +AFTER UPDATE OF salary ON employees +FOR EACH ROW EXECUTE FUNCTION log_salary_change(); + +CREATE OR REPLACE FUNCTION get_employee_details(emp_id employees.id%Type) +RETURNS public.employees.name%Type AS $$ +DECLARE + employee_name employees.name%TYPE; +BEGIN + SELECT name INTO employee_name FROM employees WHERE id = emp_id; + RETURN employee_name; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION list_high_earners(threshold NUMERIC) RETURNS VOID AS $$ +DECLARE + emp_name employees.name%TYPE; + emp_salary employees.salary%TYPE; +BEGIN + FOR emp_name, emp_salary IN + SELECT name, salary FROM employees WHERE salary > threshold + LOOP + RAISE NOTICE 'Employee: %, Salary: %', emp_name, emp_salary; + END LOOP; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION copy_high_earners(threshold NUMERIC) RETURNS VOID AS $$ +DECLARE + temp_salary employees.salary%TYPE; +BEGIN + CREATE TEMP TABLE temp_high_earners AS + SELECT * FROM employees WHERE salary > threshold; + + FOR temp_salary IN SELECT salary FROM temp_high_earners LOOP + RAISE NOTICE 'High earner salary: %', temp_salary; + END LOOP; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql index 4879281eb3..cdb2bdcff6 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/procedures/procedure.sql @@ -149,3 +149,27 @@ BEGIN RAISE NOTICE 'Employee % of age % added successfully.', emp_name, emp_age; END; $$; + +CREATE OR REPLACE PROCEDURE update_salary(emp_id INT, increment NUMERIC) AS $$ +DECLARE + current_salary employees.salary%TYPE; -- Matches the type of the salary column +BEGIN + SELECT salary INTO current_salary FROM employees WHERE id = emp_id; + + IF current_salary IS NULL THEN + RAISE NOTICE 'Employee ID % does not exist.', emp_id; + ELSE + UPDATE employees SET salary = current_salary + increment WHERE id = emp_id; + END IF; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE PROCEDURE get_employee_details_proc(emp_id employees.id%Type, salary employees.salary%TYPE, tax_rate numeric) AS $$ +DECLARE + employee_name employees.name%TYPE; +BEGIN + SELECT name INTO employee_name FROM employees e WHERE e.id = emp_id and e.salary = salary and e.tax_rate = tax_rate; + +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 0c1dc18255..431ab83cf4 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1584,5 +1584,145 @@ "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "public.get_employeee_salary", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "calculate_tax", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.tax_rate%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "log_salary_change", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "log_salary_change", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "get_employee_details", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.name%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "get_employee_details", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.id%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "list_high_earners", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.name%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "list_high_earners", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "copy_high_earners", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "update_salary", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "get_employee_details_proc", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.name%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "get_employee_details_proc", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.id%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "PROCEDURE", + "ObjectName": "get_employee_details_proc", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "employees.salary%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "get_employee_details", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "public.employees.name%TYPE", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 8b633466df..09b4ec3f28 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -36,15 +36,15 @@ }, { "ObjectType": "FUNCTION", - "TotalCount": 1, - "InvalidCount": 1, - "ObjectNames": "create_and_populate_tables" + "TotalCount": 7, + "InvalidCount": 7, + "ObjectNames": "create_and_populate_tables, public.get_employeee_salary, get_employee_details, calculate_tax, log_salary_change, list_high_earners, copy_high_earners" }, { "ObjectType": "PROCEDURE", - "TotalCount": 6, - "InvalidCount": 4, - "ObjectNames": "foo, foo1, sp_createnachabatch, test, test1, add_employee" + "TotalCount": 8, + "InvalidCount": 6, + "ObjectNames": "foo, foo1, sp_createnachabatch, test, get_employee_details_proc, test1, add_employee, update_salary" }, { "ObjectType": "VIEW", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index e3aa333e28..e87e9cb5e2 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -55,10 +55,9 @@ }, { "ObjectType": "FUNCTION", - "TotalCount": 9, - "InvalidCount": 2, - "ObjectNames": "public.process_order, schema2.process_order, public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.total" - }, + "TotalCount": 10, + "InvalidCount": 3, + "ObjectNames": "public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.process_combined_tbl, public.process_order, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.process_order, schema2.total" }, { "ObjectType": "AGGREGATE", "TotalCount": 2, @@ -67,9 +66,9 @@ }, { "ObjectType": "PROCEDURE", - "TotalCount": 2, - "InvalidCount": 0, - "ObjectNames": "public.tt_insert_data, schema2.tt_insert_data" + "TotalCount": 3, + "InvalidCount": 1, + "ObjectNames": "public.tt_insert_data, public.update_combined_tbl_data, schema2.tt_insert_data" }, { "ObjectType": "VIEW", @@ -2131,6 +2130,22 @@ } ], "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "Referenced type declaration of variables", + "Objects": [ + { + "ObjectType": "FUNCTION", + "ObjectName": "public.process_combined_tbl", + "SqlStatement": "public.combined_tbl.maddr%TYPE" + }, + { + "ObjectType": "PROCEDURE", + "ObjectName": "public.update_combined_tbl_data", + "SqlStatement": "public.combined_tbl.maddr%TYPE" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, { "FeatureName": "Advisory Locks", "Objects": [ diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index c826c33aaf..e3ef11bb0c 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -294,3 +294,56 @@ END; $$ LANGUAGE plpgsql; select process_order(1); + +-- In PG migration from pg_dump the function parameters and return will never have the %TYPE syntax, instead they have the actual type in the DDLs +-- e.g. for the below function this will be the export one `CREATE FUNCTION public.process_combined_tbl(p_id integer, p_c cidr, p_bitt bit, p_inds3 interval) RETURNS macaddr` +CREATE OR REPLACE FUNCTION public.process_combined_tbl( + p_id public.combined_tbl.id%TYPE, + p_c public.combined_tbl.c%TYPE, + p_bitt public.combined_tbl.bitt%TYPE, + p_inds3 public.combined_tbl.inds3%TYPE +) +RETURNS public.combined_tbl.maddr%TYPE AS +$$ +DECLARE + v_maddr public.combined_tbl.maddr%TYPE; +BEGIN + -- Example logic: Assigning local variable using passed-in parameter + v_maddr := p_c::text; -- Example conversion (cidr to macaddr), just for illustration + + -- Processing the passed parameters + RAISE NOTICE 'Processing: ID = %, CIDR = %, BIT = %, Interval = %, MAC = %', + p_id, p_c, p_bitt, p_inds3, v_maddr; + + -- Returning a value of the macaddr type (this could be more meaningful logic) + RETURN v_maddr; -- Returning a macaddr value +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE public.update_combined_tbl_data( + p_id public.combined_tbl.id%TYPE, + p_c public.combined_tbl.c%TYPE, + p_bitt public.combined_tbl.bitt%TYPE, + p_d public.combined_tbl.d%TYPE +) +AS +$$ +DECLARE + v_new_mac public.combined_tbl.maddr%TYPE; +BEGIN + -- Example: Using a local variable to store a macaddr value (for illustration) + v_new_mac := '00:14:22:01:23:45'::macaddr; + + -- Updating the table with provided parameters + UPDATE public.combined_tbl + SET + c = p_c, -- Updating cidr type column + bitt = p_bitt, -- Updating bit column + d = p_d, -- Updating daterange column + maddr = v_new_mac -- Using the local macaddr variable in update + WHERE id = p_id; + + RAISE NOTICE 'Updated record with ID: %, CIDR: %, BIT: %, Date range: %', + p_id, p_c, p_bitt, p_d; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index e977afb880..ae0b93f687 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -36,7 +36,7 @@ { "ObjectType": "FUNCTION", "TotalCount": 139, - "InvalidCount": 0, + "InvalidCount": 16, "ObjectNames": "mgd.acc_accession_delete, mgd.acc_assignj, mgd.acc_assignmgi, mgd.acc_delete_byacckey, mgd.acc_insert, mgd.acc_setmax, mgd.acc_split, mgd.acc_update, mgd.accref_insert, mgd.accref_process, mgd.all_allele_delete, mgd.all_allele_insert, mgd.all_allele_update, mgd.all_cellline_delete, mgd.all_cellline_update1, mgd.all_cellline_update2, mgd.all_convertallele, mgd.all_createwildtype, mgd.all_insertallele, mgd.all_mergeallele, mgd.all_mergewildtypes, mgd.all_reloadlabel, mgd.all_variant_delete, mgd.bib_keepwfrelevance, mgd.bib_refs_delete, mgd.bib_refs_insert, mgd.bib_reloadcache, mgd.bib_updatewfstatusap, mgd.bib_updatewfstatusgo, mgd.bib_updatewfstatusgxd, mgd.bib_updatewfstatusqtl, mgd.gxd_addcelltypeset, mgd.gxd_addemapaset, mgd.gxd_addgenotypeset, mgd.gxd_allelepair_insert, mgd.gxd_antibody_delete, mgd.gxd_antibody_insert, mgd.gxd_antigen_delete, mgd.gxd_antigen_insert, mgd.gxd_assay_delete, mgd.gxd_assay_insert, mgd.gxd_checkduplicategenotype, mgd.gxd_gelrow_insert, mgd.gxd_genotype_delete, mgd.gxd_genotype_insert, mgd.gxd_getgenotypesdatasets, mgd.gxd_getgenotypesdatasetscount, mgd.gxd_htexperiment_delete, mgd.gxd_htrawsample_delete, mgd.gxd_htsample_ageminmax, mgd.gxd_index_insert, mgd.gxd_index_insert_before, mgd.gxd_index_update, mgd.gxd_orderallelepairs, mgd.gxd_ordergenotypes, mgd.gxd_ordergenotypesall, mgd.gxd_ordergenotypesmissing, mgd.gxd_removebadgelband, mgd.gxd_replacegenotype, mgd.img_image_delete, mgd.img_image_insert, mgd.img_setpdo, mgd.mgi_addsetmember, mgd.mgi_checkemapaclipboard, mgd.mgi_cleannote, mgd.mgi_deleteemapaclipboarddups, mgd.mgi_insertreferenceassoc, mgd.mgi_insertsynonym, mgd.mgi_mergerelationship, mgd.mgi_organism_delete, mgd.mgi_organism_insert, mgd.mgi_processnote, mgd.mgi_reference_assoc_delete, mgd.mgi_reference_assoc_insert, mgd.mgi_relationship_delete, mgd.mgi_resetageminmax, mgd.mgi_resetsequencenum, mgd.mgi_setmember_emapa_insert, mgd.mgi_setmember_emapa_update, mgd.mgi_statistic_delete, mgd.mgi_updatereferenceassoc, mgd.mgi_updatesetmember, mgd.mld_expt_marker_update, mgd.mld_expts_delete, mgd.mld_expts_insert, mgd.mrk_allelewithdrawal, mgd.mrk_cluster_delete, mgd.mrk_copyhistory, mgd.mrk_deletewithdrawal, mgd.mrk_inserthistory, mgd.mrk_marker_delete, mgd.mrk_marker_insert, mgd.mrk_marker_update, mgd.mrk_mergewithdrawal, mgd.mrk_reloadlocation, mgd.mrk_reloadreference, mgd.mrk_simplewithdrawal, mgd.mrk_strainmarker_delete, mgd.mrk_updatekeys, mgd.prb_ageminmax, mgd.prb_getstrainbyreference, mgd.prb_getstraindatasets, mgd.prb_getstrainreferences, mgd.prb_insertreference, mgd.prb_marker_insert, mgd.prb_marker_update, mgd.prb_mergestrain, mgd.prb_probe_delete, mgd.prb_probe_insert, mgd.prb_processanonymoussource, mgd.prb_processprobesource, mgd.prb_processseqloadersource, mgd.prb_processsequencesource, mgd.prb_reference_delete, mgd.prb_reference_update, mgd.prb_setstrainreview, mgd.prb_source_delete, mgd.prb_strain_delete, mgd.prb_strain_insert, mgd.seq_deletebycreatedby, mgd.seq_deletedummy, mgd.seq_deleteobsoletedummy, mgd.seq_merge, mgd.seq_sequence_delete, mgd.seq_split, mgd.voc_annot_insert, mgd.voc_copyannotevidencenotes, mgd.voc_evidence_delete, mgd.voc_evidence_insert, mgd.voc_evidence_property_delete, mgd.voc_evidence_update, mgd.voc_mergeannotations, mgd.voc_mergedupannotations, mgd.voc_mergeterms, mgd.voc_processannotheader, mgd.voc_processannotheaderall, mgd.voc_processannotheadermissing, mgd.voc_resetterms, mgd.voc_term_delete" }, { @@ -13513,5 +13513,232 @@ } ], "UnsupportedQueryConstructs": null, - "UnsupportedPlPgSqlObjects": null + "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "Referenced type declaration of variables", + "Objects": [ + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_assignmgi", + "SqlStatement": "acc_accession.accid%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_assignmgi", + "SqlStatement": "acc_accession.accid%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_insert", + "SqlStatement": "acc_accession.prefixPart%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_update", + "SqlStatement": "acc_accession.prefixPart%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_update", + "SqlStatement": "acc_accession.prefixPart%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_createwildtype", + "SqlStatement": "all_allele.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_insertallele", + "SqlStatement": "all_allele.isextinct%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_insertallele", + "SqlStatement": "all_allele.ismixed%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_reloadlabel", + "SqlStatement": "all_label.label%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_reloadlabel", + "SqlStatement": "all_label.labelType%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_reloadlabel", + "SqlStatement": "all_label.labelTypeName%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.img_setpdo", + "SqlStatement": "acc_accession.accID%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_allelewithdrawal", + "SqlStatement": "mrk_marker.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_allelewithdrawal", + "SqlStatement": "mrk_marker.name%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_allelewithdrawal", + "SqlStatement": "mrk_marker.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_copyhistory", + "SqlStatement": "mrk_history.name%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_copyhistory", + "SqlStatement": "mrk_history.event_date%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_deletewithdrawal", + "SqlStatement": "mrk_marker.name%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.name%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.chromosome%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.name%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.cytogeneticOffset%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "mrk_marker.cmOffset%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "SqlStatement": "all_allele.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "mrk_marker.chromosome%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "mrk_marker.cytogeneticOffset%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "mrk_marker.cmoffset%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "seq_coord_cache.startCoordinate%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "seq_coord_cache.endCoordinate%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "seq_coord_cache.strand%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "voc_term.term%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "map_coord_collection.abbreviation%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "seq_coord_cache.version%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "SqlStatement": "seq_coord_cache.chromosome%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_simplewithdrawal", + "SqlStatement": "mrk_marker.name%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_simplewithdrawal", + "SqlStatement": "mrk_marker.symbol%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_ageminmax", + "SqlStatement": "prb_source.age%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_ageminmax", + "SqlStatement": "prb_source.age%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_mergestrain", + "SqlStatement": "acc_accession.accID%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_mergestrain", + "SqlStatement": "acc_accession.accID%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.seq_split", + "SqlStatement": "acc_accession.accID%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.seq_split", + "SqlStatement": "acc_accession.accID%TYPE" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + } + ] } diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index ccadb13896..e6cc902c91 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -36,7 +36,7 @@ { "ObjectType": "FUNCTION", "TotalCount": 139, - "InvalidCount": 0, + "InvalidCount": 16, "ObjectNames": "mgd.acc_accession_delete, mgd.acc_assignj, mgd.acc_assignmgi, mgd.acc_delete_byacckey, mgd.acc_insert, mgd.acc_setmax, mgd.acc_split, mgd.acc_update, mgd.accref_insert, mgd.accref_process, mgd.all_allele_delete, mgd.all_allele_insert, mgd.all_allele_update, mgd.all_cellline_delete, mgd.all_cellline_update1, mgd.all_cellline_update2, mgd.all_convertallele, mgd.all_createwildtype, mgd.all_insertallele, mgd.all_mergeallele, mgd.all_mergewildtypes, mgd.all_reloadlabel, mgd.all_variant_delete, mgd.bib_keepwfrelevance, mgd.bib_refs_delete, mgd.bib_refs_insert, mgd.bib_reloadcache, mgd.bib_updatewfstatusap, mgd.bib_updatewfstatusgo, mgd.bib_updatewfstatusgxd, mgd.bib_updatewfstatusqtl, mgd.gxd_addcelltypeset, mgd.gxd_addemapaset, mgd.gxd_addgenotypeset, mgd.gxd_allelepair_insert, mgd.gxd_antibody_delete, mgd.gxd_antibody_insert, mgd.gxd_antigen_delete, mgd.gxd_antigen_insert, mgd.gxd_assay_delete, mgd.gxd_assay_insert, mgd.gxd_checkduplicategenotype, mgd.gxd_gelrow_insert, mgd.gxd_genotype_delete, mgd.gxd_genotype_insert, mgd.gxd_getgenotypesdatasets, mgd.gxd_getgenotypesdatasetscount, mgd.gxd_htexperiment_delete, mgd.gxd_htrawsample_delete, mgd.gxd_htsample_ageminmax, mgd.gxd_index_insert, mgd.gxd_index_insert_before, mgd.gxd_index_update, mgd.gxd_orderallelepairs, mgd.gxd_ordergenotypes, mgd.gxd_ordergenotypesall, mgd.gxd_ordergenotypesmissing, mgd.gxd_removebadgelband, mgd.gxd_replacegenotype, mgd.img_image_delete, mgd.img_image_insert, mgd.img_setpdo, mgd.mgi_addsetmember, mgd.mgi_checkemapaclipboard, mgd.mgi_cleannote, mgd.mgi_deleteemapaclipboarddups, mgd.mgi_insertreferenceassoc, mgd.mgi_insertsynonym, mgd.mgi_mergerelationship, mgd.mgi_organism_delete, mgd.mgi_organism_insert, mgd.mgi_processnote, mgd.mgi_reference_assoc_delete, mgd.mgi_reference_assoc_insert, mgd.mgi_relationship_delete, mgd.mgi_resetageminmax, mgd.mgi_resetsequencenum, mgd.mgi_setmember_emapa_insert, mgd.mgi_setmember_emapa_update, mgd.mgi_statistic_delete, mgd.mgi_updatereferenceassoc, mgd.mgi_updatesetmember, mgd.mld_expt_marker_update, mgd.mld_expts_delete, mgd.mld_expts_insert, mgd.mrk_allelewithdrawal, mgd.mrk_cluster_delete, mgd.mrk_copyhistory, mgd.mrk_deletewithdrawal, mgd.mrk_inserthistory, mgd.mrk_marker_delete, mgd.mrk_marker_insert, mgd.mrk_marker_update, mgd.mrk_mergewithdrawal, mgd.mrk_reloadlocation, mgd.mrk_reloadreference, mgd.mrk_simplewithdrawal, mgd.mrk_strainmarker_delete, mgd.mrk_updatekeys, mgd.prb_ageminmax, mgd.prb_getstrainbyreference, mgd.prb_getstraindatasets, mgd.prb_getstrainreferences, mgd.prb_insertreference, mgd.prb_marker_insert, mgd.prb_marker_update, mgd.prb_mergestrain, mgd.prb_probe_delete, mgd.prb_probe_insert, mgd.prb_processanonymoussource, mgd.prb_processprobesource, mgd.prb_processseqloadersource, mgd.prb_processsequencesource, mgd.prb_reference_delete, mgd.prb_reference_update, mgd.prb_setstrainreview, mgd.prb_source_delete, mgd.prb_strain_delete, mgd.prb_strain_insert, mgd.seq_deletebycreatedby, mgd.seq_deletedummy, mgd.seq_deleteobsoletedummy, mgd.seq_merge, mgd.seq_sequence_delete, mgd.seq_split, mgd.voc_annot_insert, mgd.voc_copyannotevidencenotes, mgd.voc_evidence_delete, mgd.voc_evidence_insert, mgd.voc_evidence_property_delete, mgd.voc_evidence_update, mgd.voc_mergeannotations, mgd.voc_mergedupannotations, mgd.voc_mergeterms, mgd.voc_processannotheader, mgd.voc_processannotheaderall, mgd.voc_processannotheadermissing, mgd.voc_resetterms, mgd.voc_term_delete" }, { @@ -735,6 +735,490 @@ "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_assignmgi", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accid%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_assignmgi", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accid%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_insert", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.prefixPart%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_update", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.prefixPart%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.acc_update", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.prefixPart%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_createwildtype", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_allele.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_insertallele", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_allele.isextinct%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_insertallele", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_allele.ismixed%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_reloadlabel", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_label.label%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_reloadlabel", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_label.labelType%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.all_reloadlabel", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_label.labelTypeName%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.img_setpdo", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accID%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_allelewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_allelewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.name%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_allelewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_copyhistory", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_history.name%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_copyhistory", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_history.event_date%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_deletewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.name%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.name%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.chromosome%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.name%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.cytogeneticOffset%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.cmOffset%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_mergewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "all_allele.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.chromosome%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.cytogeneticOffset%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.cmoffset%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "seq_coord_cache.startCoordinate%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "seq_coord_cache.endCoordinate%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "seq_coord_cache.strand%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "voc_term.term%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "map_coord_collection.abbreviation%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "seq_coord_cache.version%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_reloadlocation", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "seq_coord_cache.chromosome%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_simplewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.name%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mrk_simplewithdrawal", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "mrk_marker.symbol%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_ageminmax", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "prb_source.age%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_ageminmax", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "prb_source.age%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_mergestrain", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accID%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.prb_mergestrain", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accID%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.seq_split", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accID%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.seq_split", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "acc_accession.accID%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" } ] } diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 976e905ba3..9b71172dcd 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1724,6 +1724,8 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance issue.IssueInstance, fileN DocsLink: issueInstance.DocsLink, FilePath: fileName, IssueType: UNSUPPORTED_PLPGSQL_OBEJCTS, + Suggestion: issueInstance.Suggestion, + GH: issueInstance.GH, } } diff --git a/yb-voyager/src/issue/constants.go b/yb-voyager/src/issue/constants.go index fdd94c0ea2..f792d32fea 100644 --- a/yb-voyager/src/issue/constants.go +++ b/yb-voyager/src/issue/constants.go @@ -18,9 +18,10 @@ package issue // Types const ( - ADVISORY_LOCKS = "ADVISORY_LOCKS" - SYSTEM_COLUMNS = "SYSTEM_COLUMNS" - XML_FUNCTIONS = "XML_FUNCTIONS" + ADVISORY_LOCKS = "ADVISORY_LOCKS" + SYSTEM_COLUMNS = "SYSTEM_COLUMNS" + XML_FUNCTIONS = "XML_FUNCTIONS" + REFERENCED_TYPE_DECLARATION = "REFERENCED_TYPE_DECLARATION" ) // Object types diff --git a/yb-voyager/src/issue/ddl.go b/yb-voyager/src/issue/ddl.go new file mode 100644 index 0000000000..390ee1b76c --- /dev/null +++ b/yb-voyager/src/issue/ddl.go @@ -0,0 +1,31 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package issue + +var percentTypeSyntax = Issue{ + Type: REFERENCED_TYPE_DECLARATION, + TypeName: "Referenced type declaration of variables", + TypeDescription: "", + Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", + GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", +} + +func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + return newIssueInstance(percentTypeSyntax, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 9772e3a160..691a407ec6 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -18,6 +18,7 @@ package queryissue import ( "fmt" + "strings" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -83,6 +84,12 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, issues = append(issues, issuesInQuery...) } + percentTypeSyntaxIssues, err := p.GetPercentTypeSyntaxIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting reference TYPE syntax issues: %v", err) + } + issues = append(issues, percentTypeSyntaxIssues...) + return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object //e.g. replacing the DML_QUERY and "" to FUNCTION and @@ -110,7 +117,6 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, i.ObjectName = objName return i }), nil - } issues, err := p.getDMLIssues(query) @@ -120,6 +126,36 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, return issues, nil } +func (p *ParserIssueDetector) GetPercentTypeSyntaxIssues(query string) ([]issue.IssueInstance, error) { + parseTree, err := queryparser.Parse(query) + if err != nil { + return nil, fmt.Errorf("error parsing the query-%s: %v", query, err) + } + + objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) + typeNames, err := queryparser.GetAllTypeNamesInPlpgSQLStmt(query) + if err != nil { + return nil, fmt.Errorf("error getting type names in PLPGSQL: %v", err) + } + + /* + Caveats of GetAllTypeNamesInPlpgSQLStmt(): + 1. Not returning typename for variables in function parameter from this function (in correct in json as UNKNOWN), for that using the GetTypeNamesFromFuncParameters() + 2. Not returning the return type from this function (not available in json), for that using the GetReturnTypeOfFunc() + */ + if queryparser.IsFunctionObject(parseTree) { + typeNames = append(typeNames, queryparser.GetReturnTypeOfFunc(parseTree)) + } + typeNames = append(typeNames, queryparser.GetFuncParametersTypeNames(parseTree)...) + var issues []issue.IssueInstance + for _, typeName := range typeNames { + if strings.HasSuffix(typeName, "%TYPE") { + issues = append(issues, issue.NewPercentTypeSyntaxIssue(objType, objName, typeName)) // TODO: confirm + } + } + return issues, nil +} + func (p *ParserIssueDetector) GetDMLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { issues, err := p.getDMLIssues(query) if err != nil { diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/queryparser/query_parser.go index 247a30c597..2a4737cd7a 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/queryparser/query_parser.go @@ -17,6 +17,7 @@ package queryparser import ( "fmt" + "strings" pg_query "github.com/pganalyze/pg_query_go/v5" "github.com/samber/lo" @@ -65,7 +66,6 @@ func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect. return parseTree.Stmts[0].Stmt.ProtoReflect() } - func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) @@ -163,4 +163,77 @@ func getCreateViewNode(parseTree *pg_query.ParseResult) (*pg_query.Node_ViewStmt func getCreateFuncStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateFunctionStmt, bool) { node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) return node, ok -} \ No newline at end of file +} + +func IsFunctionObject(parseTree *pg_query.ParseResult) bool { + funcNode, ok := getCreateFuncStmtNode(parseTree) + if !ok { + return false + } + return !funcNode.CreateFunctionStmt.IsProcedure +} + +/* +return type ex- +CREATE OR REPLACE FUNCTION public.process_combined_tbl( + + ... + +) +RETURNS public.combined_tbl.maddr%TYPE AS +return_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} names:{string:{sval:"maddr"}} +pct_type:true typemod:-1 location:226} +*/ +func GetReturnTypeOfFunc(parseTree *pg_query.ParseResult) string { + funcNode, _ := getCreateFuncStmtNode(parseTree) + returnType := funcNode.CreateFunctionStmt.GetReturnType() + return convertParserTypeNameToString(returnType) +} + +func getQualifiedTypeName(typeNames []*pg_query.Node) string { + var typeNameStrings []string + for _, n := range typeNames { + typeNameStrings = append(typeNameStrings, n.GetString_().Sval) + } + return strings.Join(typeNameStrings, ".") +} + +func convertParserTypeNameToString(typeVar *pg_query.TypeName) string { + typeNames := typeVar.GetNames() + finalTypeName := getQualifiedTypeName(typeNames) // type name can qualified table_name.column in case of %TYPE + if typeVar.PctType { // %TYPE declaration, so adding %TYPE for using it further + return finalTypeName + "%TYPE" + } + return finalTypeName +} + +/* +function ex - +CREATE OR REPLACE FUNCTION public.process_combined_tbl( + + p_id int, + p_c public.combined_tbl.c%TYPE, + p_bitt public.combined_tbl.bitt%TYPE, + .. + +) +parseTree- +parameters:{function_parameter:{name:"p_id" arg_type:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:66} +mode:FUNC_PARAM_DEFAULT}} parameters:{function_parameter:{name:"p_c" arg_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} +names:{string:{sval:"c"}} pct_type:true typemod:-1 location:87} mode:FUNC_PARAM_DEFAULT}} parameters:{function_parameter:{name:"p_bitt" +arg_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} names:{string:{sval:"bitt"}} pct_type:true typemod:-1 +location:136} mode:FUNC_PARAM_DEFAULT}} +*/ +func GetFuncParametersTypeNames(parseTree *pg_query.ParseResult) []string { + funcNode, _ := getCreateFuncStmtNode(parseTree) + parameters := funcNode.CreateFunctionStmt.GetParameters() + var paramTypeNames []string + for _, param := range parameters { + funcParam, ok := param.Node.(*pg_query.Node_FunctionParameter) + if ok { + paramType := funcParam.FunctionParameter.ArgType + paramTypeNames = append(paramTypeNames, convertParserTypeNameToString(paramType)) + } + } + return paramTypeNames +} diff --git a/yb-voyager/src/queryparser/traversal_plpgsql.go b/yb-voyager/src/queryparser/traversal_plpgsql.go index 2fa757930c..bc6a3c419e 100644 --- a/yb-voyager/src/queryparser/traversal_plpgsql.go +++ b/yb-voyager/src/queryparser/traversal_plpgsql.go @@ -28,6 +28,11 @@ const ( QUERY = "query" ACTION = "action" + DATUMS = "datums" + PLPGSQL_VAR = "PLpgSQL_var" + DATATYPE = "datatype" + TYPENAME = "typname" + PLPGSQL_TYPE = "PLpgSQL_type" PLPGSQL_FUNCTION = "PLpgSQL_function" ) @@ -49,32 +54,15 @@ These issues are majorly expressions, conditions, assignments, loop variables, r * */ func GetAllPLPGSQLStatements(query string) ([]string, error) { - parsedJson, err := ParsePLPGSQLToJson(query) + parsedJson, parsedJsonMap, err := getParsedJsonMap(query) if err != nil { - log.Infof("error in parsing the stmt-%s to json: %v", query, err) return []string{}, err } - if parsedJson == "" { - return []string{}, nil - } - var parsedJsonMapList []map[string]interface{} - //Refer to the queryparser.traversal_plpgsql.go for example and sample parsed json - log.Debugf("parsing the json string-%s of stmt-%s", parsedJson, query) - err = json.Unmarshal([]byte(parsedJson), &parsedJsonMapList) - if err != nil { - return []string{}, fmt.Errorf("error parsing the json string of stmt-%s: %v", query, err) - } - - if len(parsedJsonMapList) == 0 { - return []string{}, nil - } - - parsedJsonMap := parsedJsonMapList[0] function := parsedJsonMap[PLPGSQL_FUNCTION] parsedFunctionMap, ok := function.(map[string]interface{}) if !ok { - return []string{}, fmt.Errorf("error getting the PlPgSQL_Function field in parsed json-%s", parsedJson) + return []string{}, fmt.Errorf("the PlPgSQL_Function field is not a map in parsed json-%s", parsedJson) } actions := parsedFunctionMap[ACTION] @@ -228,3 +216,194 @@ func formatExprQuery(q string) string { } return q } + +func getParsedJsonMap(query string) (string, map[string]interface{}, error) { + parsedJson, err := ParsePLPGSQLToJson(query) + if err != nil { + log.Infof("error in parsing the stmt-%s to json: %v", query, err) + return parsedJson, nil, err + } + if parsedJson == "" { + return "", nil, nil + } + var parsedJsonMapList []map[string]interface{} + //Refer to the queryparser.traversal_plpgsql.go for example and sample parsed json + log.Debugf("parsing the json string-%s of stmt-%s", parsedJson, query) + err = json.Unmarshal([]byte(parsedJson), &parsedJsonMapList) + if err != nil { + return parsedJson, nil, fmt.Errorf("error parsing the json string of stmt-%s: %v", query, err) + } + + if len(parsedJsonMapList) == 0 { + return parsedJson, nil, nil + } + + return parsedJson, parsedJsonMapList[0], nil +} + +/* +example - +CREATE FUNCTION public.get_employeee_salary(emp_id employees.employee_id%TYPE) RETURNS employees.salary%Type + + LANGUAGE plpgsql + AS $$ + +DECLARE + + emp_salary employees.salary%TYPE; + +BEGIN + + SELECT salary INTO emp_salary + FROM employees + WHERE employee_id = emp_id; + RETURN emp_salary; + +END; +$$; +[ + + { + "PLpgSQL_function": { + "datums": [ + { + "PLpgSQL_var": { + "refname": "emp_id", + "datatype": { + "PLpgSQL_type": { + "typname": "UNKNOWN" + } + } + } + }, + { + "PLpgSQL_var": { + "refname": "found", + "datatype": { + "PLpgSQL_type": { + "typname": "UNKNOWN" + } + } + } + }, + { + "PLpgSQL_var": { + "refname": "emp_salary", + "lineno": 3, + "datatype": { + "PLpgSQL_type": { + "typname": "employees.salary%TYPE" + } + } + } + }, + { + "PLpgSQL_row": { + "refname": "(unnamed row)", + "lineno": 5, + "fields": [ + { + "name": "emp_salary", + "varno": 2 + } + ] + } + } + ],"action": { + .... + } + } + }, + + Caveats: + 1. Not returning typename for variables in function parameter from this function (in correct in json as UNKNOWN), for that using the GetTypeNamesFromFuncParameters() + 2. Not returning the return type from this function (not available in json), for that using the GetReturnTypeOfFunc() +*/ +func GetAllTypeNamesInPlpgSQLStmt(query string) ([]string, error) { + parsedJson, parsedJsonMap, err := getParsedJsonMap(query) + if err != nil { + return []string{}, nil + } + function := parsedJsonMap[PLPGSQL_FUNCTION] + parsedFunctionMap, ok := function.(map[string]interface{}) + if !ok { + return []string{}, fmt.Errorf("the PlPgSQL_Function field is not a map in parsed json-%s", parsedJson) + } + + datums := parsedFunctionMap[DATUMS] + datumList, isList := datums.([]interface{}) + if !isList { + return []string{}, fmt.Errorf("type names datums field is not list in parsed json-%s", parsedJson) + } + + var typeNames []string + for _, datum := range datumList { + datumMap, ok := datum.(map[string]interface{}) + if !ok { + log.Errorf("datum is not a map-%v", datum) + continue + } + for key, val := range datumMap { + switch key { + case PLPGSQL_VAR: + typeName, err := getTypeNameFromPlpgSQLVar(val) + if err != nil { + log.Errorf("not able to get typename from PLPGSQL_VAR(%v): %v", val, err) + continue + } + typeNames = append(typeNames, typeName) + } + } + } + return typeNames, nil +} + +/* +example of PLPGSQL_VAR - + + "PLpgSQL_var": { + "refname": "tax_rate", + "lineno": 3, + "datatype": { + "PLpgSQL_type": { + "typname": "employees.tax_rate%TYPE" + } + } + } +*/ +func getTypeNameFromPlpgSQLVar(plpgsqlVar interface{}) (string, error) { + //getting the map of of PLpgSQL_Var json + valueMap, ok := plpgsqlVar.(map[string]interface{}) + if !ok { + return "", fmt.Errorf("PLPGSQL_VAR is not a map-%v", plpgsqlVar) + } + + //getting the "datatype" field of PLpgSQL_Var json + datatype, ok := valueMap[DATATYPE] + if !ok { + return "", fmt.Errorf("datatype is not in the PLPGSQL_VAR map-%v", valueMap) + } + + datatypeValueMap, ok := datatype.(map[string]interface{}) + if !ok { + return "", fmt.Errorf("datatype is not a map-%v", datatype) + } + + plpgsqlType, ok := datatypeValueMap[PLPGSQL_TYPE] + if !ok { + return "", fmt.Errorf("PLPGSQL_Type is not in the datatype map-%v", datatypeValueMap) + } + + typeValueMap, ok := plpgsqlType.(map[string]interface{}) + if !ok { + return "", fmt.Errorf("PLPGSQL_Type is not a map-%v", plpgsqlType) + } + + typeName, ok := typeValueMap[TYPENAME] + if !ok { + return "", fmt.Errorf("typname is not in the PLPGSQL_Type map-%v", typeValueMap) + } + + return typeName.(string), nil + +} From 9ba1ffabd394aadc8ade3fb6358180fe0d01d127 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 5 Dec 2024 10:56:09 +0530 Subject: [PATCH 031/105] Report target-db-version and supported versions in assessment/analyze-schema reports (#2007) Changes in assessment and analyze reports to specify target-db-version and future supported versions for unsupported features/issues. --- migtests/scripts/functions.sh | 1 + .../tests/analyze-schema/expected_issues.json | 528 ++++++++++++------ .../expectedAssessmentReport.json | 15 +- .../expectedChild1AssessmentReport.json | 15 +- .../expectedChild2AssessmentReport.json | 15 +- .../expectedAssessmentReport.json | 119 ++-- .../expected_schema_analysis_report.json | 249 ++++++--- .../expectedAssessmentReport.json | 204 ++++--- .../expectedAssessmentReport.json | 118 ++-- .../expected_schema_analysis_report.json | 318 +++++++---- .../expectedAssessmentReport.json | 126 +++-- .../expected_schema_analysis_report.json | 120 ++-- .../expectedAssessmentReport.json | 116 ++-- .../expected_schema_analysis_report.json | 9 +- .../expectedAssessmentReport.json | 121 ++-- .../expected_schema_analysis_report.json | 9 +- .../expectedAssessmentReport.json | 116 ++-- .../expected_schema_analysis_report.json | 330 +++++++---- .../expectedAssessmentReport.json | 121 ++-- .../expected_schema_analysis_report.json | 24 +- .../expectedAssessmentReport.json | 121 ++-- .../expected_schema_analysis_report.json | 6 +- .../expectedAssessmentReport.json | 116 ++-- .../expected_schema_analysis_report.json | 132 +++-- yb-voyager/cmd/analyzeSchema.go | 39 +- yb-voyager/cmd/assessMigrationCommand.go | 85 ++- yb-voyager/cmd/common.go | 12 +- yb-voyager/cmd/importSchema.go | 2 +- .../migration_assessment_report.template | 23 +- .../cmd/templates/schema_analysis_report.html | 5 + .../cmd/templates/schema_analysis_report.txt | 5 + yb-voyager/src/issue/issue.go | 3 + yb-voyager/src/utils/commonVariables.go | 48 +- yb-voyager/src/ybversion/constants.go | 4 +- yb-voyager/src/ybversion/yb_version.go | 16 + yb-voyager/src/ybversion/yb_version_test.go | 3 + 36 files changed, 2146 insertions(+), 1148 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 5b10574cc3..f8b707bcb3 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -879,6 +879,7 @@ normalize_json() { if type == "object" then .ObjectNames? |= (if type == "string" then split(", ") | sort | join(", ") else . end) | .VoyagerVersion? = "IGNORED" | + .TargetDBVersion? = "IGNORED" | .DbVersion? = "IGNORED" | .FilePath? = "IGNORED" | .OptimalSelectConnectionsPerNode? = "IGNORED" | diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 431ab83cf4..af94b1944e 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -7,7 +7,8 @@ "SqlStatement": "CREATE INDEX film_fulltext_idx ON public.film USING gist (fulltext);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1337" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1337", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -16,7 +17,8 @@ "Reason": "JSON_ARRAYAGG() function is not available in YugabyteDB", "SqlStatement": "CREATE MATERIALIZED VIEW test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", "Suggestion": "Rename the function to YugabyteDB's equivalent JSON_AGG()", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1542" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -26,7 +28,8 @@ "SqlStatement": "CREATE index idx1 on combined_tbl (c);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -36,7 +39,8 @@ "SqlStatement": "CREATE index idx2 on combined_tbl (ci);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -46,7 +50,8 @@ "SqlStatement": "CREATE index idx3 on combined_tbl (b);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -56,7 +61,8 @@ "SqlStatement": "CREATE index idx4 on combined_tbl (j);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -66,7 +72,8 @@ "SqlStatement": "CREATE index idx5 on combined_tbl (l);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -76,7 +83,8 @@ "SqlStatement": "CREATE index idx6 on combined_tbl (ls);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -86,7 +94,8 @@ "SqlStatement": "CREATE index idx7 on combined_tbl (maddr);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -96,7 +105,8 @@ "SqlStatement": "CREATE index idx8 on combined_tbl (maddr8);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -106,7 +116,8 @@ "SqlStatement": "CREATE index idx9 on combined_tbl (p);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -116,7 +127,8 @@ "SqlStatement": "CREATE index idx10 on combined_tbl (lsn);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -126,7 +138,8 @@ "SqlStatement": "CREATE index idx11 on combined_tbl (p1);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -136,7 +149,8 @@ "SqlStatement": "CREATE index idx12 on combined_tbl (p2);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -146,7 +160,8 @@ "SqlStatement": "CREATE index idx13 on combined_tbl (id1);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -156,7 +171,8 @@ "SqlStatement": "CREATE INDEX idx14 on combined_tbl (bitt);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -166,7 +182,8 @@ "SqlStatement": "CREATE INDEX idx15 on combined_tbl (bittv);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -176,7 +193,8 @@ "SqlStatement": "CREATE TABLE public.documents (\n id integer NOT NULL,\n title_tsvector tsvector,\n content_tsvector tsvector,\n\tlist_of_sections text[]\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -186,7 +204,8 @@ "SqlStatement": "CREATE TABLE public.documents (\n id integer NOT NULL,\n title_tsvector tsvector,\n content_tsvector tsvector,\n\tlist_of_sections text[]\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -196,7 +215,8 @@ "SqlStatement": "CREATE TABLE public.ts_query_table (\n id int generated by default as identity,\n query tsquery\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -206,7 +226,8 @@ "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -216,7 +237,8 @@ "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[]\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -226,7 +248,8 @@ "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -236,7 +259,8 @@ "SqlStatement": "CREATE INDEX idx_udt on test_udt(home_address);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -246,7 +270,8 @@ "SqlStatement": "CREATE INDEX idx_udt1 on test_udt(home_address1);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -256,7 +281,8 @@ "SqlStatement": "CREATE INDEX \"idx\u0026_enum2\" on test_udt((some_field::non_public.enum_test));", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -266,7 +292,8 @@ "SqlStatement": "CREATE INDEX abc ON public.example USING btree (new_id) WITH (fillfactor='70'); ", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "Suggestion": "Remove the storage parameters from the DDL", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -276,7 +303,8 @@ "SqlStatement": "CREATE INDEX abc ON schema2.example USING btree (new_id) WITH (fillfactor='70'); ", "Suggestion": "Remove the storage parameters from the DDL", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -286,7 +314,8 @@ "SqlStatement": "CREATE OR REPLACE PROCEDURE foo (p_id integer) AS $body$\nBEGIN\n drop temporary table if exists temp;\n create temporary table temp(id int, name text);\n insert into temp(id,name) select id,p_name from bar where p_id=id;\n select name from temp;\nend;\n$body$\nLANGUAGE PLPGSQL\nSECURITY DEFINER\n;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#drop-temporary-table-statements-are-not-supported", "Suggestion": "remove \"temporary\" and change it to \"drop table\"", - "GH": "https://github.com/yugabyte/yb-voyager/issues/705" + "GH": "https://github.com/yugabyte/yb-voyager/issues/705", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -296,7 +325,8 @@ "SqlStatement": "CREATE TABLE sales (\n\tcust_id bigint NOT NULL,\n\tname varchar(40),\n\tstore_id varchar(20) NOT NULL,\n\tbill_no bigint NOT NULL,\n\tbill_date timestamp NOT NULL,\n\tamount decimal(8,2) NOT NULL,\n\tPRIMARY KEY (bill_date)\n) PARTITION BY RANGE (extract(year from date(bill_date))) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", "Suggestion": "Remove the Constriant from the table definition", - "GH": "https://github.com/yugabyte/yb-voyager/issues/698" + "GH": "https://github.com/yugabyte/yb-voyager/issues/698", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -306,7 +336,8 @@ "SqlStatement": "CREATE TABLE salaries2 (\n\temp_no bigint NOT NULL,\n\tsalary bigint NOT NULL,\n\tfrom_date timestamp NOT NULL,\n\tto_date timestamp NOT NULL,\n\tPRIMARY KEY (emp_no,from_date)\n) PARTITION BY RANGE (extract(epoch from date(from_date))) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", "Suggestion": "Remove the Constriant from the table definition", - "GH": "https://github.com/yugabyte/yb-voyager/issues/698" + "GH": "https://github.com/yugabyte/yb-voyager/issues/698", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -315,7 +346,8 @@ "Reason": "ALTER VIEW not supported yet.", "SqlStatement": "ALTER VIEW view_name TO select * from test;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1131" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1131", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -324,7 +356,8 @@ "Reason": "Unsupported PG syntax - 'syntax error at or near \"TO\"'", "SqlStatement": "ALTER VIEW view_name TO select * from test;", "Suggestion": "Fix the schema as per PG syntax", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1625" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1625", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -333,7 +366,8 @@ "Reason": "ALTER TABLE OF not supported yet.", "SqlStatement": "Alter table only party_profile_part of parent_tbl add constraint party_profile_pk primary key (party_profile_id);", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -343,7 +377,8 @@ "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", "Suggestion": "Make it a single column partition by list or choose other supported Partitioning methods", - "GH": "https://github.com/yugabyte/yb-voyager/issues/699" + "GH": "https://github.com/yugabyte/yb-voyager/issues/699", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -353,7 +388,8 @@ "SqlStatement": "CREATE TABLE test_2 (\n\tid numeric NOT NULL PRIMARY KEY,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50)\n) PARTITION BY LIST (country_code, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", "Suggestion": "Make it a single column partition by list or choose other supported Partitioning methods", - "GH": "https://github.com/yugabyte/yb-voyager/issues/699" + "GH": "https://github.com/yugabyte/yb-voyager/issues/699", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -363,7 +399,8 @@ "SqlStatement": "CREATE TABLE test_2 (\n\tid numeric NOT NULL PRIMARY KEY,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50)\n) PARTITION BY LIST (country_code, record_type) ;", "Suggestion": "Add all Partition columns to Primary Key", "GH": "https://github.com/yugabyte/yb-voyager/issues/578", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -373,7 +410,8 @@ "SqlStatement": "CREATE TABLE test_5 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY RANGE (country_code, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", "Suggestion": "Add all Partition columns to Primary Key", - "GH": "https://github.com/yugabyte/yb-voyager/issues/578" + "GH": "https://github.com/yugabyte/yb-voyager/issues/578", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -383,7 +421,8 @@ "SqlStatement": "CREATE TABLE test_6 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id,country_code)\n) PARTITION BY RANGE (country_code, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", "Suggestion": "Add all Partition columns to Primary Key", - "GH": "https://github.com/yugabyte/yb-voyager/issues/578" + "GH": "https://github.com/yugabyte/yb-voyager/issues/578", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -393,7 +432,8 @@ "SqlStatement": "CREATE TABLE test_7 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id,country_code)\n) PARTITION BY RANGE (descriptions, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", "Suggestion": "Add all Partition columns to Primary Key", - "GH": "https://github.com/yugabyte/yb-voyager/issues/578" + "GH": "https://github.com/yugabyte/yb-voyager/issues/578", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -403,7 +443,8 @@ "SqlStatement": "CREATE TABLE test_8 (\n\torder_id bigint NOT NULL,\n\torder_date timestamp,\n\torder_mode varchar(8),\n\tcustomer_id integer,\n\torder_mode smallint,\n\torder_total double precision,\n\tsales_rep_id integer,\n\tpromotion_id integer,\n\tPRIMARY KEY (order_id,order_mode,customer_id,order_total,sales_rep_id)\n) PARTITION BY RANGE (promotion_id, order_date, sales_rep_id) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", "Suggestion": "Add all Partition columns to Primary Key", - "GH": "https://github.com/yugabyte/yb-voyager/issues/578" + "GH": "https://github.com/yugabyte/yb-voyager/issues/578", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -413,7 +454,8 @@ "SqlStatement": "CREATE TABLE test_non_pk_multi_column_list (\n\tid numeric NOT NULL PRIMARY KEY,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50)\n) PARTITION BY LIST (country_code, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", "Suggestion": "Make it a single column partition by list or choose other supported Partitioning methods", - "GH": "https://github.com/yugabyte/yb-voyager/issues/699" + "GH": "https://github.com/yugabyte/yb-voyager/issues/699", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -423,7 +465,8 @@ "SqlStatement": "CREATE TABLE test_non_pk_multi_column_list (\n\tid numeric NOT NULL PRIMARY KEY,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50)\n) PARTITION BY LIST (country_code, record_type) ;", "Suggestion": "Add all Partition columns to Primary Key", "GH": "https://github.com/yugabyte/yb-voyager/issues/578", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -433,7 +476,8 @@ "SqlStatement": "CREATE CONVERSION myconv FOR 'UTF8' TO 'LATIN1' FROM myfunc;", "Suggestion": "Remove it from the exported schema", "GH": "https://github.com/yugabyte/yugabyte-db/issues/10866", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -443,7 +487,8 @@ "SqlStatement": "ALTER CONVERSION myconv rename to my_conv_1;", "Suggestion": "Remove it from the exported schema", "GH": "https://github.com/YugaByte/yugabyte-db/issues/10866", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -453,7 +498,8 @@ "SqlStatement": "CREATE INDEX idx_name1 ON table_name USING spgist (col1);", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1337", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -463,7 +509,8 @@ "SqlStatement": "CREATE INDEX idx_name3 ON schema_name.table_name USING gin (col1,col2,col3);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gin-indexes-on-multiple-columns-are-not-supported", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10652" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10652", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -473,7 +520,8 @@ "SqlStatement": "CREATE UNLOGGED TABLE tbl_unlogged (id int, val text);", "Suggestion": "Remove UNLOGGED keyword to make it work", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1129/", - "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported" + "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -483,7 +531,8 @@ "SqlStatement": "CREATE VIEW v1 AS SELECT * FROM t1 WHERE a \u003c 2\nWITH CHECK OPTION;", "Suggestion": "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -493,7 +542,8 @@ "SqlStatement": "CREATE VIEW v2 AS SELECT * FROM t1 WHERE a \u003c 2\nWITH LOCAL CHECK OPTION;", "Suggestion": "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -502,7 +552,8 @@ "Reason": "DROP multiple objects not supported yet.", "SqlStatement": "DROP COLLATION IF EXISTS coll1,coll2,coll3;", "Suggestion": "DROP COLLATION coll1;DROP COLLATION coll2;DROP COLLATION coll3;", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/880" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/880", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -511,7 +562,8 @@ "Reason": "DROP multiple objects not supported yet.", "SqlStatement": "DROP INDEX idx1,idx2,idx3;", "Suggestion": "DROP INDEX idx1;DROP INDEX idx2;DROP INDEX idx3;", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/880" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/880", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -520,7 +572,8 @@ "Reason": "DROP multiple objects not supported yet.", "SqlStatement": "DROP VIEW IF EXISTS view1,view2,view3;", "Suggestion": "DROP VIEW view1;DROP VIEW view2;DROP VIEW view3;", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/880" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/880", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -529,7 +582,8 @@ "Reason": "DROP multiple objects not supported yet.", "SqlStatement": "DROP SEQUENCE seq1_tbl,seq2_tbl,seq3_tbl;", "Suggestion": "DROP SEQUENCE seq1_tbl;DROP SEQUENCE seq2_tbl;DROP SEQUENCE seq3_tbl;", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/880" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/880", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -538,7 +592,8 @@ "Reason": "DROP INDEX CONCURRENTLY not supported yet", "SqlStatement": "DROP INDEX CONCURRENTLY sales_quantity_index;", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/22717" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/22717", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -548,7 +603,8 @@ "SqlStatement": "CREATE TRIGGER before_insert_or_delete_row_trigger\nBEFORE INSERT OR DELETE ON public.range_columns_partition_test\nFOR EACH ROW\nEXECUTE FUNCTION handle_insert_or_delete();", "Suggestion": "Create the triggers on individual partitions.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/24830", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -558,7 +614,8 @@ "SqlStatement": "CREATE TRIGGER transfer_insert\n AFTER INSERT ON transfer\n REFERENCING NEW TABLE AS inserted\n FOR EACH STATEMENT\n EXECUTE FUNCTION check_transfer_balances_to_zero();", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1668", - "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers" + "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -568,7 +625,8 @@ "SqlStatement": "CREATE CONSTRAINT TRIGGER some_trig\n AFTER DELETE ON xyz_schema.abc\n DEFERRABLE INITIALLY DEFERRED\n FOR EACH ROW EXECUTE PROCEDURE xyz_schema.some_trig();", "Suggestion": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1709" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1709", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -577,7 +635,8 @@ "Reason": "CREATE ACCESS METHOD is not supported.", "SqlStatement": "CREATE ACCESS METHOD heptree TYPE INDEX HANDLER heptree_handler;", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10693" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10693", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -586,7 +645,8 @@ "Reason": "REINDEX is not supported.", "SqlStatement": "REINDEX TABLE my_table;", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10267" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10267", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -596,7 +656,8 @@ "SqlStatement": "CREATE TABLE public.employees4 (\n id integer NOT NULL,\n first_name character varying(50) NOT NULL,\n last_name character varying(50) NOT NULL,\n full_name character varying(101) GENERATED ALWAYS AS ((((first_name)::text || ' '::text) || (last_name)::text)) STORED\n);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", "Suggestion": "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -606,7 +667,8 @@ "SqlStatement": "CREATE TABLE order_details (\n detail_id integer NOT NULL,\n quantity integer,\n price_per_unit numeric,\n amount numeric GENERATED ALWAYS AS (((quantity)::numeric * price_per_unit)) STORED\n);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", "Suggestion": "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -615,7 +677,8 @@ "Reason": "LIKE clause not supported yet.", "SqlStatement": "CREATE TABLE table_xyz\n (LIKE xyz INCLUDING DEFAULTS INCLUDING CONSTRAINTS);", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -624,7 +687,8 @@ "Reason": "LIKE ALL is not supported yet.", "SqlStatement": "CREATE TABLE table_abc\n (LIKE abc INCLUDING ALL);", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10697" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10697", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -634,7 +698,8 @@ "SqlStatement": "CREATE TABLE table_1 () INHERITS (xyz);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -643,7 +708,8 @@ "Reason": "OIDs are not supported for user tables.", "SqlStatement": "Create table table_test (col1 text, col2 int) with (OIDS = TRUE);", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10273" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10273", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -653,7 +719,8 @@ "SqlStatement": "CREATE INDEX idx1 on combined_tbl1 (d);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -663,7 +730,8 @@ "SqlStatement": "CREATE INDEX idx2 on combined_tbl1 (t);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -673,7 +741,8 @@ "SqlStatement": "CREATE INDEX idx3 on combined_tbl1 (tz);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -683,7 +752,8 @@ "SqlStatement": "CREATE INDEX idx4 on combined_tbl1 (n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -693,7 +763,8 @@ "SqlStatement": "CREATE INDEX idx5 on combined_tbl1 (i4);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -703,7 +774,8 @@ "SqlStatement": "CREATE INDEX idx6 on combined_tbl1 (i8);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -713,7 +785,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -723,7 +796,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -733,7 +807,8 @@ "SqlStatement": "ALTER TABLE combined_tbl\n\t\tADD CONSTRAINT combined_tbl_unique UNIQUE(id, bitt);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -743,7 +818,8 @@ "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -753,7 +829,8 @@ "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -763,7 +840,8 @@ "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -773,7 +851,8 @@ "SqlStatement": "ALTER TABLE combined_tbl1\n\t\tADD CONSTRAINT combined_tbl1_unique UNIQUE(id, d);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -783,7 +862,8 @@ "SqlStatement": "CREATE INDEX idx7 on combined_tbl1 (inym);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -793,7 +873,8 @@ "SqlStatement": "CREATE INDEX idx8 on combined_tbl1 (inds);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -803,7 +884,8 @@ "SqlStatement": "create table test_interval(\n frequency interval primary key,\n\tcol1 int\n);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -812,7 +894,8 @@ "Reason": "ALTER TABLE SET SCHEMA not supported yet.", "SqlStatement": "ALTER TABLE oldschema.tbl_name SET SCHEMA newschema;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/3947" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/3947", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -821,7 +904,8 @@ "Reason": "CREATE SCHEMA with elements not supported yet.", "SqlStatement": "CREATE SCHEMA hollywood\n CREATE TABLE films (title text, release date, awards text[])\n CREATE VIEW winners AS\n SELECT title, release FROM films WHERE awards IS NOT NULL;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/10865" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/10865", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -830,7 +914,8 @@ "Reason": "ALTER TABLE ALTER column SET STATISTICS not supported yet.", "SqlStatement": "alter table table_name alter column column_name set statistics 100;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -839,7 +924,8 @@ "Reason": "ALTER TABLE ALTER column SET STORAGE not supported yet.", "SqlStatement": "alter table test alter column col set STORAGE EXTERNAL;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -849,7 +935,8 @@ "SqlStatement": "alter table test_1 alter column col1 set (attribute_option=value);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "Suggestion": "Remove it from the exported schema", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1124" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -859,7 +946,8 @@ "SqlStatement": "alter table test DISABLE RULE example_rule;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "Suggestion": "Remove this and the rule 'example_rule' from the exported schema to be not enabled on the table.", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1124" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -869,7 +957,8 @@ "SqlStatement": "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor='70');", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "Suggestion": "Remove the storage parameters from the DDL", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -879,7 +968,8 @@ "SqlStatement": "alter table abc cluster on xyz;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "Suggestion": "Remove it from the exported schema.", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -889,7 +979,8 @@ "SqlStatement": "CREATE INDEX tsvector_idx ON public.documents (title_tsvector, id);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -899,7 +990,8 @@ "SqlStatement": "CREATE INDEX tsquery_idx ON public.ts_query_table (query);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -909,7 +1001,8 @@ "SqlStatement": "CREATE INDEX idx_citext ON public.citext_type USING btree (data);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -919,7 +1012,8 @@ "SqlStatement": "CREATE INDEX idx_inet ON public.inet_type USING btree (data);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -929,7 +1023,8 @@ "SqlStatement": "CREATE INDEX idx_json ON public.test_jsonb (data);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -939,7 +1034,8 @@ "SqlStatement": "CREATE INDEX idx_json2 ON public.test_jsonb ((data2::jsonb));", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -949,7 +1045,8 @@ "SqlStatement": "create index idx_array on public.documents (list_of_sections);", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -958,7 +1055,8 @@ "Reason": "ALTER TABLE SET WITHOUT CLUSTER not supported yet.", "SqlStatement": "alter table test SET WITHOUT CLUSTER;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -967,7 +1065,8 @@ "Reason": "ALTER INDEX SET not supported yet.", "SqlStatement": "ALTER INDEX abc set TABLESPACE new_tbl;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -976,7 +1075,8 @@ "Reason": "ALTER TABLE INHERIT not supported yet.", "SqlStatement": "alter table test_3 INHERIT test_2;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -985,7 +1085,8 @@ "Reason": "ALTER TABLE VALIDATE CONSTRAINT not supported yet.", "SqlStatement": "ALTER TABLE distributors VALIDATE CONSTRAINT distfk;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -995,7 +1096,8 @@ "SqlStatement": "ALTER TABLE abc\nADD CONSTRAINT cnstr_id\n UNIQUE (id)\nDEFERRABLE;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", "Suggestion": "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1005,7 +1107,8 @@ "SqlStatement": "ALTER TABLE ONLY public.users\n ADD CONSTRAINT users_email_key UNIQUE (email) DEFERRABLE;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", "Suggestion": "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1015,7 +1118,8 @@ "SqlStatement": "create table unique_def_test(id int UNIQUE DEFERRABLE, c1 int);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", "Suggestion": "Remove these constraints from the exported schema and make the necessary changes to the application before pointing it to target", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1025,7 +1129,8 @@ "SqlStatement": "create table unique_def_test1(id int, c1 int, UNIQUE(id) DEFERRABLE INITIALLY DEFERRED);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", "Suggestion": "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -1035,7 +1140,8 @@ "SqlStatement": "CREATE TABLE test_xml_type(id int, data xml);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -1045,7 +1151,8 @@ "SqlStatement": "CREATE TABLE test_xid_type(id int, data xid);", "Suggestion": "Functions for this type e.g. txid_current are not supported in YugabyteDB yet", "GH": "https://github.com/yugabyte/yugabyte-db/issues/15638", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -1055,7 +1162,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -1065,7 +1173,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -1075,7 +1184,8 @@ "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326)\n );", "Suggestion": "", "GH": "https://github.com/yugabyte/yugabyte-db/issues/11323", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1085,7 +1195,8 @@ "SqlStatement": "CREATE FOREIGN TABLE public.locations ( id integer NOT NULL, name character varying(100), geom geometry(Point,4326) ) SERVER remote_server OPTIONS ( schema_name 'public', table_name 'remote_locations' ); ", "Suggestion": "SERVER 'remote_server', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -1095,7 +1206,8 @@ "SqlStatement": "CREATE FOREIGN TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326)\n ) SERVER remote_server\nOPTIONS (\n schema_name 'public',\n table_name 'remote_locations'\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yugabyte-db/issues/11323", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1105,7 +1217,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1115,7 +1228,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1125,7 +1239,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1135,7 +1250,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1145,7 +1261,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1155,7 +1272,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1165,7 +1283,8 @@ "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1175,7 +1294,8 @@ "SqlStatement": "ALTER TABLE ONLY public.meeting\n ADD CONSTRAINT no_time_overlap EXCLUDE USING gist (room_id WITH =, time_range WITH \u0026\u0026);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", "Suggestion": "Refer docs link for details on possible workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1185,7 +1305,8 @@ "SqlStatement": "CREATE TABLE \"Test\"(\n\tid int,\n\troom_id int,\n\ttime_range trange,\n\troomid int,\n\ttimerange tsrange,\n\tEXCLUDE USING gist (room_id WITH =, time_range WITH \u0026\u0026),\n\tCONSTRAINT no_time_overlap_constr EXCLUDE USING gist (roomid WITH =, timerange WITH \u0026\u0026)\n);", "Suggestion": "Refer docs link for details on possible workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1195,7 +1316,8 @@ "SqlStatement": "CREATE TABLE \"Test\"(\n\tid int,\n\troom_id int,\n\ttime_range trange,\n\troomid int,\n\ttimerange tsrange,\n\tEXCLUDE USING gist (room_id WITH =, time_range WITH \u0026\u0026),\n\tCONSTRAINT no_time_overlap_constr EXCLUDE USING gist (roomid WITH =, timerange WITH \u0026\u0026)\n);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", "Suggestion": "Refer docs link for details on possible workaround", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1205,7 +1327,8 @@ "SqlStatement": "ALTER TABLE ONLY public.range_columns_partition_test\n ADD CONSTRAINT range_columns_partition_test_pkey PRIMARY KEY (a, b);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10074" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10074", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1214,7 +1337,8 @@ "Reason": "Primary key constraints are not supported on foreign tables.", "SqlStatement": "CREATE FOREIGN TABLE tbl_p(\n\tid int PRIMARY KEY\n) SERVER remote_server\nOPTIONS (\n schema_name 'public',\n table_name 'remote_table'\n);", "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10698" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10698", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1223,7 +1347,8 @@ "Reason": "COMPOUND TRIGGER not supported in YugabyteDB.", "SqlStatement": "CREATE TRIGGER emp_trig\n\tCOMPOUND INSERT ON emp FOR EACH ROW\n\tEXECUTE PROCEDURE trigger_fct_emp_trig();", "Suggestion": "", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1543" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1543", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1232,7 +1357,8 @@ "Reason": "ALTER TYPE not supported yet.", "SqlStatement": "ALTER TYPE colors ADD VALUE 'orange' AFTER 'red';", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1893" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1893", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1241,7 +1367,8 @@ "Reason": "ALTER TYPE not supported yet.", "SqlStatement": "ALTER TYPE compfoo ADD ATTRIBUTE f3 int;", "Suggestion": "", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/1893" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/1893", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1250,7 +1377,8 @@ "Reason": "AnyData datatype doesn't have a mapping in YugabyteDB", "SqlStatement": "CREATE TABLE anydata_test (\n\tid numeric,\n\tcontent ANYDATA\n) ;", "Suggestion": "Remove the column with AnyData datatype or change it to a relevant supported datatype", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1541" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1541", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1259,7 +1387,8 @@ "Reason": "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "SqlStatement": "CREATE TABLE anydataset_test (\n\tid numeric,\n\tcontent ANYDATASET\n) ;", "Suggestion": "Remove the column with AnyDataSet datatype or change it to a relevant supported datatype", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1541" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1541", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1268,7 +1397,8 @@ "Reason": "AnyType datatype doesn't have a mapping in YugabyteDB", "SqlStatement": "CREATE TABLE anytype_test (\n\tid numeric,\n\tcontent ANYTYPE\n) ;", "Suggestion": "Remove the column with AnyType datatype or change it to a relevant supported datatype", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1541" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1541", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1277,7 +1407,8 @@ "Reason": "URIType datatype doesn't have a mapping in YugabyteDB", "SqlStatement": "CREATE TABLE uritype_test (\n\tid numeric,\n\tcontent URITYPE\n) ;", "Suggestion": "Remove the column with URIType datatype or change it to a relevant supported datatype", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1541" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1541", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1286,7 +1417,8 @@ "Reason": "JSON_ARRAYAGG() function is not available in YugabyteDB", "SqlStatement": "CREATE OR REPLACE view test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", "Suggestion": "Rename the function to YugabyteDB's equivalent JSON_AGG()", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1542" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1295,7 +1427,8 @@ "Reason": "JSON_ARRAYAGG() function is not available in YugabyteDB", "SqlStatement": "CREATE OR REPLACE PROCEDURE foo1 (p_id integer) AS $body$\nBEGIN\n create temporary table temp(id int, agg bigint);\n insert into temp(id,agg) select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg FROM test1\n select agg from temp;\nend;\n$body$\nLANGUAGE PLPGSQL\nSECURITY DEFINER\n;", "Suggestion": "Rename the function to YugabyteDB's equivalent JSON_AGG()", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1542" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1304,7 +1437,8 @@ "Reason": "JSON_ARRAYAGG() function is not available in YugabyteDB", "SqlStatement": "CREATE TRIGGER test\n INSTEAD OF INSERT on test for each ROW\n EXECUTE PROCEDURE JSON_ARRAYAGG(trunc(b, 2) order by t desc);", "Suggestion": "Rename the function to YugabyteDB's equivalent JSON_AGG()", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1542" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1314,7 +1448,8 @@ "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", "Suggestion": "Add all Partition columns to Primary Key", "GH": "https://github.com/yugabyte/yb-voyager/issues/578", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1324,7 +1459,8 @@ "SqlStatement": "CREATE TABLE sales_data (\n sales_id numeric NOT NULL,\n sales_date timestamp,\n sales_amount numeric,\n PRIMARY KEY (sales_id)\n) PARTITION BY RANGE (sales_date) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", "Suggestion": "Add all Partition columns to Primary Key", - "GH": "https://github.com/yugabyte/yb-voyager/issues/578" + "GH": "https://github.com/yugabyte/yb-voyager/issues/578", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1334,7 +1470,8 @@ "SqlStatement": "CREATE EXTENSION IF NOT EXISTS aws_commons WITH SCHEMA public;", "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", "Suggestion": "", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1538" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1538", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1344,7 +1481,8 @@ "SqlStatement": "CREATE EXTENSION IF NOT EXISTS plperl WITH SCHEMA pg_catalog;", "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", "Suggestion": "", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1538" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1538", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1354,7 +1492,8 @@ "SqlStatement": "CREATE EXTENSION IF NOT EXISTS hstore_plperl WITH SCHEMA public;", "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", "Suggestion": "", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1538" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1538", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1363,7 +1502,8 @@ "Reason": "This FETCH clause might not be supported yet", "SqlStatement": "CREATE OR REPLACE PROCEDURE test () AS $body$\nDECLARE\n cur CURSOR FOR SELECT column_name FROM table_name;\n row RECORD;\nBEGIN\n OPEN cur;\n FETCH PRIOR FROM cur INTO row;\n CLOSE cur;\nEND;\n$body$\nLANGUAGE PLPGSQL\nSECURITY DEFINER\n;", "Suggestion": "Please verify the DDL on your YugabyteDB version before proceeding", - "GH": "https://github.com/YugaByte/yugabyte-db/issues/6514" + "GH": "https://github.com/YugaByte/yugabyte-db/issues/6514", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1373,7 +1513,8 @@ "SqlStatement": "CREATE FOREIGN TABLE tbl_p( \tid int PRIMARY KEY ) SERVER remote_server OPTIONS ( schema_name 'public', table_name 'remote_table' ); ", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", "Suggestion": "SERVER 'remote_server', and USER MAPPING should be created manually on the target to create and use the foreign table", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1627" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1383,7 +1524,8 @@ "SqlStatement": "CREATE TABLE enum_example.bugs (\n id integer NOT NULL,\n description text,\n status enum_example.bug_status,\n _status enum_example.bug_status GENERATED ALWAYS AS (status) STORED,\n severity enum_example.bug_severity,\n _severity enum_example.bug_severity GENERATED ALWAYS AS (severity) STORED,\n info enum_example.bug_info GENERATED ALWAYS AS (enum_example.make_bug_info(status, severity)) STORED\n);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", "Suggestion": "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695" + "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1393,7 +1535,8 @@ "SqlStatement": "CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true);", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -1403,7 +1546,8 @@ "SqlStatement": "CREATE POLICY p1 ON z1 TO regress_rls_group1 USING (a % 2 = 0);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1655" + "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1413,7 +1557,8 @@ "SqlStatement": "SELECT pg_advisory_lock(sender_id);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1423,7 +1568,8 @@ "SqlStatement": "SELECT pg_advisory_lock(receiver_id);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1433,7 +1579,8 @@ "SqlStatement": "SELECT pg_advisory_unlock(sender_id);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1443,7 +1590,8 @@ "SqlStatement": "SELECT pg_advisory_unlock(receiver_id);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1453,7 +1601,8 @@ "SqlStatement": "SELECT pg_advisory_unlock(sender_id);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1463,7 +1612,8 @@ "SqlStatement": "SELECT pg_advisory_unlock(receiver_id);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1473,7 +1623,8 @@ "SqlStatement": "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1483,7 +1634,8 @@ "SqlStatement": "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1493,7 +1645,8 @@ "SqlStatement": "SELECT id, first_name FROM employees WHERE pg_try_advisory_lock(300) IS TRUE;", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1503,7 +1656,8 @@ "SqlStatement": "SELECT e.id, e.name,\n ROW_NUMBER() OVER (ORDER BY e.ctid) AS row_num\n FROM employees e;", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1513,7 +1667,8 @@ "SqlStatement": "SELECT e.id, x.employee_xml\n FROM employees e\n JOIN (\n SELECT xmlelement(name \"employee\", xmlattributes(e.id AS \"id\"), e.name) AS employee_xml\n FROM employees e\n ) x ON x.employee_xml IS NOT NULL\n WHERE xmlexists('//employee[name=\"John Doe\"]' PASSING BY REF x.employee_xml);", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1523,7 +1678,8 @@ "SqlStatement": "SELECT e.id,\n CASE\n WHEN e.salary \u003e 100000 THEN pg_advisory_lock(e.id)\n ELSE pg_advisory_unlock(e.id)\n END AS lock_status\n FROM employees e;", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1533,7 +1689,8 @@ "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1543,7 +1700,8 @@ "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1553,7 +1711,8 @@ "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1563,7 +1722,8 @@ "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1573,7 +1733,8 @@ "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1583,7 +1744,8 @@ "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1593,7 +1755,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1603,7 +1766,8 @@ "SqlStatement": "employees.tax_rate%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1613,7 +1777,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1623,7 +1788,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1633,7 +1799,8 @@ "SqlStatement": "employees.name%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1643,7 +1810,8 @@ "SqlStatement": "employees.id%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1653,7 +1821,8 @@ "SqlStatement": "employees.name%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1663,7 +1832,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1673,7 +1843,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1683,7 +1854,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1693,7 +1865,8 @@ "SqlStatement": "employees.name%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1703,7 +1876,8 @@ "SqlStatement": "employees.id%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1713,7 +1887,8 @@ "SqlStatement": "employees.salary%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1723,6 +1898,7 @@ "SqlStatement": "public.employees.name%TYPE", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json index a238ee8513..d0e493bd42 100644 --- a/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json @@ -192,7 +192,8 @@ "ObjectName": "trg_simple_insert", "SqlStatement": "CREATE TRIGGER trg_simple_insert\n\tCOMPOUND INSERT ON simple_table FOR EACH ROW\n\tEXECUTE PROCEDURE trigger_fct_trg_simple_insert();" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Indexes", @@ -217,7 +218,8 @@ "ObjectName": "Index Name: REV_INDEX, Index Type=NORMAL/REV INDEX", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Virtual Columns", @@ -230,7 +232,8 @@ "ObjectName": "GENERATED_COLUMN_TABLE.TOTAL_PRICE", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited Types", @@ -239,7 +242,8 @@ "ObjectName": "SIMPLE_CAR_TYPE", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Partitioning Methods", @@ -252,7 +256,8 @@ "ObjectName": "Table Name: SALES, Partition Method: SYSTEM PARTITION", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json index 91100541e7..44e835b585 100644 --- a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json +++ b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json @@ -121,7 +121,8 @@ "ObjectName": "trg_simple_insert", "SqlStatement": "CREATE TRIGGER trg_simple_insert\n\tCOMPOUND INSERT ON simple_table FOR EACH ROW\n\tEXECUTE PROCEDURE trigger_fct_trg_simple_insert();" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Indexes", @@ -146,7 +147,8 @@ "ObjectName": "Index Name: PK_IOT_TABLE, Index Type=IOT - TOP INDEX", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Virtual Columns", @@ -159,7 +161,8 @@ "ObjectName": "GENERATED_COLUMN_TABLE.TOTAL_PRICE", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited Types", @@ -168,7 +171,8 @@ "ObjectName": "SIMPLE_CAR_TYPE", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Partitioning Methods", @@ -181,7 +185,8 @@ "ObjectName": "Table Name: EMPLOYEES2, Partition Method: REFERENCE PARTITION", "SqlStatement": "" } - ] + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json index d67d01860e..297ad902e4 100644 --- a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json +++ b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json @@ -109,23 +109,28 @@ "UnsupportedFeatures": [ { "FeatureName": "Compound Triggers", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Indexes", - "Objects": null + "Objects": null, + "MinimumVersionsFixedIn": null }, { "FeatureName": "Virtual Columns", - "Objects": null + "Objects": null, + "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited Types", - "Objects": null + "Objects": null, + "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Partitioning Methods", - "Objects": null + "Objects": null, + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 12884e9d65..842f993a55 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -311,52 +311,64 @@ "UnsupportedFeatures": [ { "FeatureName": "GIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", "Objects": [ @@ -633,39 +645,48 @@ "SqlStatement": "ALTER TABLE production.vproductanddescription CLUSTER ON ix_vproductanddescription;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1698,17 +1719,20 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, @@ -1757,7 +1781,8 @@ "SqlStatement": "SELECT store.businessentityid, store.name, unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualSales\", unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualRevenue\", unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"BankName\", unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(5) AS \"BusinessType\", unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"YearOpened\", unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"Specialty\", unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"SquareFeet\", unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Brands\", unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Internet\", unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"NumberEmployees\" FROM sales.store" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index 10b779cece..5e8900e503 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -80,7 +80,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -91,7 +92,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -102,7 +104,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -113,7 +116,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -124,7 +128,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -135,7 +140,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -146,7 +152,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -157,7 +164,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -168,7 +176,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -179,7 +188,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -190,7 +200,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -201,7 +212,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -212,7 +224,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -223,7 +236,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -234,7 +248,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -245,7 +260,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -256,7 +272,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -267,7 +284,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -278,7 +296,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -289,7 +308,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -300,7 +320,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -311,7 +332,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -322,7 +344,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -333,7 +356,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -344,7 +368,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -355,7 +380,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -366,7 +392,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -377,7 +404,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -388,7 +416,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -399,7 +428,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -410,7 +440,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -421,7 +452,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -432,7 +464,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -443,7 +476,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -454,7 +488,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -465,7 +500,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -476,7 +512,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -487,7 +524,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -498,7 +536,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -509,7 +548,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -520,7 +560,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -531,7 +572,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -542,7 +584,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -553,7 +596,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -564,7 +608,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -575,7 +620,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -586,7 +632,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -597,7 +644,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -608,7 +656,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -619,7 +668,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -630,7 +680,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -641,7 +692,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -652,7 +704,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -663,7 +716,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -674,7 +728,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -685,7 +740,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -696,7 +752,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -707,7 +764,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -718,7 +776,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -729,7 +788,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -740,7 +800,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -751,7 +812,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -762,7 +824,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -773,7 +836,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -784,7 +848,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -795,7 +860,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -806,7 +872,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -817,7 +884,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -828,7 +896,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -839,7 +908,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -850,7 +920,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -861,7 +932,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -872,7 +944,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -883,7 +956,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -894,7 +968,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -905,7 +980,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -916,7 +992,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -927,7 +1004,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -938,7 +1016,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -949,7 +1028,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -960,7 +1040,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -971,7 +1052,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -982,7 +1064,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index e87e9cb5e2..262923e0d7 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -1,5 +1,6 @@ { "VoyagerVersion": "IGNORED", + "TargetDBVersion": "IGNORED", "MigrationComplexity": "HIGH", "SchemaSummary": { "Description": "Objects that will be created on the target YugabyteDB.", @@ -284,8 +285,9 @@ "SqlStatement": "CREATE INDEX idx_point_data ON schema2.mixed_data_types_table1 USING gist (point_data);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", "Objects": [ @@ -294,8 +296,9 @@ "SqlStatement": "CREATE INDEX idx_box_data_brin ON public.mixed_data_types_table1 USING brin (box_data);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", "Objects": [ @@ -304,8 +307,9 @@ "SqlStatement": "CREATE INDEX idx_box_data_spgist ON schema2.mixed_data_types_table1 USING spgist (box_data);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", "Objects": [ @@ -318,8 +322,9 @@ "SqlStatement": "CREATE CONSTRAINT TRIGGER enforce_shipped_date_constraint AFTER UPDATE ON schema2.orders2 NOT DEFERRABLE INITIALLY IMMEDIATE FOR EACH ROW WHEN ((((new.status)::text = 'shipped'::text) AND (new.shipped_date IS NULL))) EXECUTE FUNCTION schema2.prevent_update_shipped_without_date();" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", "Objects": [ @@ -332,8 +337,9 @@ "SqlStatement": "CREATE TABLE schema2.child_table (\n specific_column1 date\n)\nINHERITS (schema2.parent_table);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", "Objects": [ @@ -346,28 +352,34 @@ "SqlStatement": "CREATE TABLE schema2.employees2 (\n id integer NOT NULL,\n first_name character varying(50) NOT NULL,\n last_name character varying(50) NOT NULL,\n full_name character varying(101) GENERATED ALWAYS AS ((((first_name)::text || ' '::text) || (last_name)::text)) STORED,\n department character varying(50)\n);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", "Objects": [ @@ -376,20 +388,24 @@ "SqlStatement": "CREATE TRIGGER before_sales_region_insert_update BEFORE INSERT OR UPDATE ON public.sales_region FOR EACH ROW EXECUTE FUNCTION public.check_sales_region();" } ], - "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables" + "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", + "MinimumVersionsFixedIn": null }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", "Objects": [ @@ -398,8 +414,9 @@ "SqlStatement": "ALTER TABLE ONLY public.test_exclude_basic\n ADD CONSTRAINT no_same_name_address EXCLUDE USING btree (name WITH =, address WITH =);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", "Objects": [ @@ -412,8 +429,9 @@ "SqlStatement": "ALTER TABLE ONLY schema2.orders2\n ADD CONSTRAINT orders2_order_number_key UNIQUE (order_number) DEFERRABLE;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", "Objects": [ @@ -426,8 +444,9 @@ "SqlStatement": "CREATE VIEW schema2.sales_employees AS\n SELECT employees2.id,\n employees2.first_name,\n employees2.last_name,\n employees2.full_name\n FROM schema2.employees2\n WHERE ((employees2.department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", "Objects": [ @@ -496,8 +515,9 @@ "SqlStatement": "CREATE INDEX idx9 ON public.combined_tbl USING btree (inds3);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", "Objects": [ @@ -514,8 +534,9 @@ "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT combined_tbl_pkey PRIMARY KEY (id, arr_enum);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", "Objects": [ @@ -528,7 +549,8 @@ "SqlStatement": "CREATE UNLOGGED TABLE schema2.tbl_unlogged (\n id integer,\n val text\n);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1949,13 +1971,15 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [ @@ -1969,8 +1993,9 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." - }, + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration", "Objects": [ @@ -2016,8 +2041,9 @@ } ], "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." - }, + "FeatureDescription": "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ @@ -2059,74 +2085,88 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": [ { "ConstructTypeName": "Advisory Locks", "Query": "SELECT pg_advisory_xact_lock($1,$2)", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "Advisory Locks", "Query": "SELECT pg_advisory_unlock($1,$2)", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "Advisory Locks", "Query": "SELECT pg_advisory_unlock_all()", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "Advisory Locks", "Query": "SELECT pg_advisory_lock($1,$2)", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "System Columns", "Query": "SELECT ctid, tableoid, xmin, xmax, cmin, cmax\nFROM employees2", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT xmlparse(document $1) as xmldata", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT table_to_xml($1, $2, $3, $4)", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT xmlforest(first_name AS element1, last_name AS element2) FROM employees2", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT xmlelement(name root, xmlelement(name child, $1))", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT xml_is_well_formed($1)", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT\n s.section_name,\n b.title,\n b.author\nFROM\n library_nested l,\n XMLTABLE(\n $1\n PASSING l.lib_data\n COLUMNS\n section_name TEXT PATH $2,\n books XML PATH $3\n ) AS s,\n XMLTABLE(\n $4\n PASSING s.books\n COLUMNS\n title TEXT PATH $5,\n author TEXT PATH $6\n) AS b", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT\n o.order_id,\n items.product,\n items.quantity::INT\nFROM\n orders_lateral o\n CROSS JOIN LATERAL XMLTABLE(\n $1\n PASSING o.order_details\n COLUMNS\n product TEXT PATH $2,\n quantity TEXT PATH $3\n) AS items", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "XML Functions", "Query": "SELECT *\nFROM xmltable(\n $1\n PASSING $2\n COLUMNS \n name TEXT PATH $3\n)", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": [ @@ -2144,7 +2184,8 @@ "SqlStatement": "public.combined_tbl.maddr%TYPE" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "FeatureName": "Advisory Locks", @@ -2165,8 +2206,9 @@ "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "XML Functions", "Objects": [ @@ -2176,8 +2218,9 @@ "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "System Columns", "Objects": [ @@ -2187,7 +2230,8 @@ "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null } ] } \ No newline at end of file diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index ae0b93f687..ee6799397f 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -250,44 +250,54 @@ "UnsupportedFeatures": [ { "FeatureName": "GIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", "Objects": [ @@ -540,47 +550,58 @@ "SqlStatement": "ALTER TABLE mgd.all_knockout_cache CLUSTER ON all_knockout_cache_idx_clustered;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -13499,18 +13520,21 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." - } + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + } ], "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": [ diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index e6cc902c91..eb4e17c785 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -63,7 +63,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -74,7 +75,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -85,7 +87,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -96,7 +99,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -107,7 +111,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -118,7 +123,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -129,7 +135,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -140,7 +147,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -151,7 +159,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -162,7 +171,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -173,7 +183,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -184,7 +195,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -195,7 +207,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -206,7 +219,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -217,7 +231,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -228,7 +243,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -239,7 +255,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -250,7 +267,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -261,7 +279,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -272,7 +291,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -283,7 +303,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -294,7 +315,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -305,7 +327,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -316,7 +339,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -327,7 +351,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -338,7 +363,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -349,7 +375,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -360,7 +387,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -371,7 +399,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -382,7 +411,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -393,7 +423,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -404,7 +435,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -415,7 +447,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -426,7 +459,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -437,7 +471,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -448,7 +483,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -459,7 +495,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -470,7 +507,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -481,7 +519,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -492,7 +531,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -503,7 +543,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -514,7 +555,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -525,7 +567,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -536,7 +579,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -547,7 +591,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -558,7 +603,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -569,7 +615,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -580,7 +627,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -591,7 +639,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -602,7 +651,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -613,7 +663,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -624,7 +675,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -635,7 +687,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -646,7 +699,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -657,7 +711,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -668,7 +723,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -679,7 +735,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -690,7 +747,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -701,7 +759,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -712,7 +771,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -723,7 +783,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -734,7 +795,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove it from the exported schema.", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1124", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -745,7 +807,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -756,7 +819,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -767,7 +831,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -778,7 +843,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -789,7 +855,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -800,7 +867,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -811,7 +879,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -822,7 +891,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -833,7 +903,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -844,7 +915,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -855,7 +927,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -866,7 +939,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -877,7 +951,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -888,7 +963,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -899,7 +975,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -910,7 +987,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -921,7 +999,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -932,7 +1011,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -943,7 +1023,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -954,7 +1035,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -965,7 +1047,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -976,7 +1059,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -987,7 +1071,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -998,7 +1083,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1009,7 +1095,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1020,7 +1107,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1031,7 +1119,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1042,7 +1131,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1053,7 +1143,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1064,7 +1155,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1075,7 +1167,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1086,7 +1179,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1097,7 +1191,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1108,7 +1203,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1119,7 +1215,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1130,7 +1227,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1141,7 +1239,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1152,7 +1251,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1163,7 +1263,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1174,7 +1275,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1185,7 +1287,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1196,7 +1299,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1207,7 +1311,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_plpgsql_objects", @@ -1218,7 +1323,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index a6acd0153c..2b49cf1ed9 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -501,20 +501,24 @@ "SqlStatement": "CREATE INDEX hidx ON extension_example.testhstore USING gist (h extension_example.gist_hstore_ops (siglen='32'));" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", "Objects": [ @@ -535,16 +539,19 @@ "SqlStatement": "CREATE TABLE regress_rls_schema.t3_3 (\n id integer NOT NULL,\n c text,\n b text,\n a integer\n)\nINHERITS (regress_rls_schema.t1_3);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", "Objects": [ @@ -557,8 +564,9 @@ "SqlStatement": "CREATE TABLE enum_example.bugs (\n id integer NOT NULL,\n description text,\n status enum_example.bug_status,\n _status enum_example.bug_status GENERATED ALWAYS AS (status) STORED,\n severity enum_example.bug_severity,\n _severity enum_example.bug_severity GENERATED ALWAYS AS (severity) STORED,\n info enum_example.bug_info GENERATED ALWAYS AS (enum_example.make_bug_info(status, severity)) STORED\n);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", "Objects": [ @@ -567,24 +575,29 @@ "SqlStatement": "CREATE CONVERSION conversion_example.myconv FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", "Objects": [ @@ -601,20 +614,24 @@ "SqlStatement": "CREATE INDEX title_idx_with_duplicates ON idx_ex.films USING btree (title) WITH (deduplicate_items=off); " } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", "Objects": [ @@ -623,12 +640,14 @@ "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT b1.a,\n b1.b\n FROM regress_rls_schema.b1\n WHERE (b1.a \u003e 0)\n WITH CASCADED CHECK OPTION;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", "Objects": [ @@ -637,11 +656,13 @@ "SqlStatement": "CREATE INDEX idx_1 ON composite_type_examples.ordinary_table USING btree (basic_);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -5510,8 +5531,9 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [ @@ -5521,8 +5543,9 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [ @@ -5580,8 +5603,9 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." - }, + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration", "Objects": [ @@ -5619,8 +5643,9 @@ } ], "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." - }, + "FeatureDescription": "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ @@ -5698,7 +5723,8 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index 2c6e452ce7..fd4eefb46d 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -150,7 +150,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -161,7 +162,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -172,7 +174,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -183,7 +186,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -194,7 +198,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -205,7 +210,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/10695", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -216,7 +222,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1337", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -227,7 +234,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Refer to the docs link for the workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25003", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -238,7 +246,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -249,7 +258,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -260,7 +270,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -271,7 +282,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -282,7 +294,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -293,7 +306,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -304,7 +318,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -315,7 +330,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -326,7 +342,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql", "Suggestion": "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -337,7 +354,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/conversions/conversion.sql", "Suggestion": "Remove it from the exported schema", "GH": "https://github.com/yugabyte/yugabyte-db/issues/10866", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -348,7 +366,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/foreign_table.sql", "Suggestion": "SERVER 'technically_this_server', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -359,7 +378,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -370,7 +390,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -381,7 +402,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -392,7 +414,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -403,7 +426,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -414,7 +438,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -425,7 +450,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -436,7 +462,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -447,7 +474,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -458,7 +486,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -469,7 +498,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -480,7 +510,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -491,7 +522,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -502,7 +534,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -513,7 +546,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -524,7 +558,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -535,7 +570,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -546,7 +582,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -557,7 +594,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -568,7 +606,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -579,7 +618,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/policies/policy.sql", "Suggestion": "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", "GH": "https://github.com/yugabyte/yb-voyager/issues/1655", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 83933b92ad..5fdd452359 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -81,60 +81,74 @@ "SqlStatement": "CREATE INDEX changeset_geom_gist ON public.osm_changeset USING gist (geom);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", "Objects": [ @@ -143,31 +157,38 @@ "SqlStatement": "CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public;" } ], - "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/" - }, + "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -304,17 +325,20 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json index b353e463aa..d4c7d50407 100755 --- a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json @@ -52,7 +52,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/osm/export-dir/schema/extensions/extension.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1538", - "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/" + "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_datatypes", @@ -63,7 +64,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/osm/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yugabyte-db/issues/11323", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -74,7 +76,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/osm/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1337", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index 53c130bfd1..4670c68ce9 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -82,87 +82,108 @@ "UnsupportedFeatures": [ { "FeatureName": "GIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -201,8 +222,9 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [ @@ -216,13 +238,15 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." - }, + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ @@ -236,7 +260,8 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json index 6d85facc5c..19ad430ca4 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json @@ -69,7 +69,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/pgtbrus/export-dir/schema/tables/foreign_table.sql", "Suggestion": "SERVER 'p10', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -80,7 +81,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/pgtbrus/export-dir/schema/tables/foreign_table.sql", "Suggestion": "SERVER 'p10', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -91,7 +93,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/pgtbrus/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 5b83660cca..560cd9403b 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -379,20 +379,24 @@ "UnsupportedFeatures": [ { "FeatureName": "GIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", "Objects": [ @@ -837,71 +841,88 @@ "SqlStatement": "CREATE TABLE rnacen.xref_p9_not_deleted (\n dbid smallint,\n created integer,\n last integer,\n upi character varying(26),\n version_i integer,\n deleted character(1),\n \"timestamp\" timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone,\n userstamp character varying(20) DEFAULT 'USER'::character varying,\n ac character varying(300),\n version integer,\n taxid bigint,\n id bigint DEFAULT nextval('rnacen.xref_pk_seq'::regclass),\n CONSTRAINT \"ck_xref$deleted\" CHECK ((deleted = ANY (ARRAY['Y'::bpchar, 'N'::bpchar]))),\n CONSTRAINT xref_p9_not_deleted_check CHECK (((dbid = 9) AND (deleted = 'N'::bpchar)))\n)\nINHERITS (rnacen.xref);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -26588,17 +26609,20 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json index d9c1c349d1..b697b742cb 100644 --- a/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json @@ -76,7 +76,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -87,7 +88,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -98,7 +100,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -109,7 +112,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -120,7 +124,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -131,7 +136,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -142,7 +148,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -153,7 +160,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -164,7 +172,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -175,7 +184,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -186,7 +196,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -197,7 +208,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -208,7 +220,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -219,7 +232,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -230,7 +244,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -241,7 +256,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -252,7 +268,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -263,7 +280,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -274,7 +292,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -285,7 +304,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -296,7 +316,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -307,7 +328,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -318,7 +340,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -329,7 +352,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -340,7 +364,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -351,7 +376,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -362,7 +388,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -373,7 +400,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -384,7 +412,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -395,7 +424,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -406,7 +436,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -417,7 +448,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -428,7 +460,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -439,7 +472,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -450,7 +484,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -461,7 +496,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -472,7 +508,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -483,7 +520,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -494,7 +532,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -505,7 +544,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -516,7 +556,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -527,7 +568,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -538,7 +580,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -549,7 +592,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -560,7 +604,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -571,7 +616,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -582,7 +628,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -593,7 +640,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -604,7 +652,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -615,7 +664,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -626,7 +676,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -637,7 +688,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -648,7 +700,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -659,7 +712,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -670,7 +724,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -681,7 +736,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -692,7 +748,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -703,7 +760,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -714,7 +772,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -725,7 +784,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -736,7 +796,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -747,7 +808,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -758,7 +820,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -769,7 +832,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -780,7 +844,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -791,7 +856,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -802,7 +868,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -813,7 +880,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -824,7 +892,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -835,7 +904,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -846,7 +916,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -857,7 +928,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -868,7 +940,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -879,7 +952,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -890,7 +964,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -901,7 +976,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -912,7 +988,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -923,7 +1000,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -934,7 +1012,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -945,7 +1024,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -956,7 +1036,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -967,7 +1048,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -978,7 +1060,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -989,7 +1072,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1000,7 +1084,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1011,7 +1096,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1022,7 +1108,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1033,7 +1120,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1044,7 +1132,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1055,7 +1144,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1066,7 +1156,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1077,7 +1168,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1088,7 +1180,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1099,7 +1192,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1110,7 +1204,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1121,7 +1216,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1132,7 +1228,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1143,7 +1240,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1154,7 +1252,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1165,7 +1264,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1176,7 +1276,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1187,7 +1288,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1198,7 +1300,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1209,7 +1312,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1220,7 +1324,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1231,7 +1336,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1242,7 +1348,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1253,7 +1360,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1264,7 +1372,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -1275,7 +1384,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/rna/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index cb4ae18aee..184099368b 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -126,28 +126,34 @@ "SqlStatement": "CREATE INDEX film_fulltext_idx ON public.film USING gist (fulltext);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", "Objects": [ @@ -176,63 +182,78 @@ "SqlStatement": "CREATE TABLE public.payment_p2007_06 (\n CONSTRAINT payment_p2007_06_payment_date_check CHECK (((payment_date \u003e= '2007-06-01 00:00:00'::timestamp without time zone) AND (payment_date \u003c '2007-07-01 00:00:00'::timestamp without time zone)))\n)\nINHERITS (public.payment);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -943,18 +964,21 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." - }, + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ @@ -964,7 +988,8 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json index d25c01a6f4..c6e31b194d 100755 --- a/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json @@ -87,7 +87,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -98,7 +99,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -109,7 +111,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -120,7 +123,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -131,7 +135,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -142,7 +147,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -153,7 +159,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1129", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -164,7 +171,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sakila/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "", "GH": "https://github.com/YugaByte/yugabyte-db/issues/1337", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 48942b5a41..cb0333a968 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -78,76 +78,94 @@ "UnsupportedFeatures": [ { "FeatureName": "GIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", "Objects": [ @@ -156,15 +174,18 @@ "SqlStatement": "ALTER TABLE ONLY public.secret_missions\n ADD CONSTRAINT cnt_solo_agent EXCLUDE USING gist (location WITH =, mission_timeline WITH \u0026\u0026);" } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -343,18 +364,21 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." - }, + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ @@ -364,7 +388,8 @@ } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", - "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json index 0b6403a615..9e336fc615 100755 --- a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json @@ -58,7 +58,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/sample-is/export-dir/schema/tables/table.sql", "Suggestion": "Refer docs link for details on possible workaround", "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", + "MinimumVersionsFixedIn": null }, { "IssueType": "migration_caveats", @@ -69,7 +70,8 @@ "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/sample-is/export-dir/schema/tables/table.sql", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null } ] } diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index c15829b277..adaf24f5e8 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -84,56 +84,69 @@ "UnsupportedFeatures": [ { "FeatureName": "GIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BRIN indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "SPGIST indexes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Inherited tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "REFERENCING clause for triggers", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Disabling rule on table", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Clustering table on index", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", "Objects": [ @@ -314,35 +327,43 @@ "SqlStatement": "CREATE INDEX votes_type_idx ON public.votes USING btree (votetypeid) WITH (fillfactor='100'); " } ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" - }, + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Exclusion constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Index on complex datatypes", - "Objects": [] - }, + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Unlogged tables", - "Objects": [] + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1263,17 +1284,20 @@ { "FeatureName": "Alter partitioned tables to add Primary Key", "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported." - }, + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Foreign tables", "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work." - }, + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Policies", "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema." + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null } ], "UnsupportedQueryConstructs": null, diff --git a/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json index 25bc7473d7..db788c7432 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json @@ -51,7 +51,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -62,7 +63,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -73,7 +75,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -84,7 +87,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -95,7 +99,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -106,7 +111,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -117,7 +123,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -128,7 +135,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -139,7 +147,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -150,7 +159,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -161,7 +171,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -172,7 +183,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -183,7 +195,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -194,7 +207,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -205,7 +219,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -216,7 +231,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -227,7 +243,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -238,7 +255,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -249,7 +267,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -260,7 +279,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -271,7 +291,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -282,7 +303,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -293,7 +315,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -304,7 +327,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -315,7 +339,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -326,7 +351,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -337,7 +363,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -348,7 +375,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -359,7 +387,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -370,7 +399,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -381,7 +411,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -392,7 +423,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -403,7 +435,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -414,7 +447,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -425,7 +459,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -436,7 +471,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -447,7 +483,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -458,7 +495,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -469,7 +507,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -480,7 +519,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -491,7 +531,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -502,7 +543,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -513,7 +555,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null }, { "IssueType": "unsupported_features", @@ -524,7 +567,8 @@ "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql" + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + "MinimumVersionsFixedIn": null } ] } diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 9b71172dcd..57483aeb76 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -2080,11 +2080,12 @@ var funcMap = template.FuncMap{ } return total }, - "split": split, + "split": split, + "getSupportedVersionString": getSupportedVersionString, } // add info to the 'reportStruct' variable and return -func analyzeSchemaInternal(sourceDBConf *srcdb.Source) utils.SchemaReport { +func analyzeSchemaInternal(sourceDBConf *srcdb.Source, detectIssues bool) utils.SchemaReport { /* NOTE: Don't create local var with name 'schemaAnalysisReport' since global one is used across all the internal functions called by analyzeSchemaInternal() @@ -2093,6 +2094,7 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source) utils.SchemaReport { schemaAnalysisReport = utils.SchemaReport{} sourceObjList = utils.GetSchemaObjectList(sourceDBConf.DBType) initializeSummaryMap() + for _, objType := range sourceObjList { var sqlInfoArr []sqlInfo filePath := utils.GetObjectFilePath(schemaDir, objType) @@ -2105,21 +2107,34 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source) utils.SchemaReport { otherFPaths = utils.GetObjectFilePath(schemaDir, "FTS_INDEX") sqlInfoArr = append(sqlInfoArr, parseSqlFileForObjectType(otherFPaths, "FTS_INDEX")...) } - if objType == "EXTENSION" { - checkExtensions(sqlInfoArr, filePath) - } - if objType == "FOREIGN TABLE" { - checkForeignTable(sqlInfoArr, filePath) - } - checker(sqlInfoArr, filePath, objType) + if detectIssues { + if objType == "EXTENSION" { + checkExtensions(sqlInfoArr, filePath) + } + if objType == "FOREIGN TABLE" { + checkForeignTable(sqlInfoArr, filePath) + } + checker(sqlInfoArr, filePath, objType) + + if objType == "CONVERSION" { + checkConversions(sqlInfoArr, filePath) + } - if objType == "CONVERSION" { - checkConversions(sqlInfoArr, filePath) + // Ideally all filtering of issues should happen in queryissue pkg layer, + // but until we move all issue detection logic to queryissue pkg, we will filter issues here as well. + schemaAnalysisReport.Issues = lo.Filter(schemaAnalysisReport.Issues, func(i utils.Issue, index int) bool { + fixed, err := i.IsFixedIn(targetDbVersion) + if err != nil { + utils.ErrExit("checking if issue %v is supported: %v", i, err) + } + return !fixed + }) } } schemaAnalysisReport.SchemaSummary = reportSchemaSummary(sourceDBConf) schemaAnalysisReport.VoyagerVersion = utils.YB_VOYAGER_VERSION + schemaAnalysisReport.TargetDBVersion = targetDbVersion schemaAnalysisReport.MigrationComplexity = getMigrationComplexity(sourceDBConf.DBType, schemaDir, schemaAnalysisReport) return schemaAnalysisReport } @@ -2171,7 +2186,7 @@ func analyzeSchema() { if err != nil { utils.ErrExit("analyze schema : load migration status record: %s", err) } - analyzeSchemaInternal(msr.SourceDBConf) + analyzeSchemaInternal(msr.SourceDBConf, true) if analyzeSchemaReportFormat != "" { generateAnalyzeSchemaReport(msr, analyzeSchemaReportFormat) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 754369b027..9ffaad7243 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -830,6 +830,9 @@ var bytesTemplate []byte func generateAssessmentReport() (err error) { utils.PrintAndLog("Generating assessment report...") + assessmentReport.VoyagerVersion = utils.YB_VOYAGER_VERSION + assessmentReport.TargetDBVersion = targetDbVersion + err = getAssessmentReportContentFromAnalyzeSchema() if err != nil { return fmt.Errorf("failed to generate assessment report content from analyze schema: %w", err) @@ -849,7 +852,6 @@ func generateAssessmentReport() (err error) { assessmentReport.UnsupportedQueryConstructs = unsupportedQueries } - assessmentReport.VoyagerVersion = utils.YB_VOYAGER_VERSION unsupportedDataTypes, unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB, err := fetchColumnsWithUnsupportedDataTypes() if err != nil { return fmt.Errorf("failed to fetch columns with unsupported data types: %w", err) @@ -881,7 +883,7 @@ func generateAssessmentReport() (err error) { } func getAssessmentReportContentFromAnalyzeSchema() error { - schemaAnalysisReport := analyzeSchemaInternal(&source) + schemaAnalysisReport := analyzeSchemaInternal(&source, true) assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary assessmentReport.SchemaSummary.Description = SCHEMA_SUMMARY_DESCRIPTION @@ -912,10 +914,35 @@ func getAssessmentReportContentFromAnalyzeSchema() error { return nil } +// when we group multiple Issue instances into a single bucket of UnsupportedFeature. +// Ideally, all the issues in the same bucket should have the same minimum version fixed in. +// We want to validate that and fail if not. +func areMinVersionsFixedInEqual(m1 map[string]*ybversion.YBVersion, m2 map[string]*ybversion.YBVersion) bool { + if m1 == nil && m2 == nil { + return true + } + if m1 == nil || m2 == nil { + return false + } + + if len(m1) != len(m2) { + return false + } + for k, v := range m1 { + if m2[k] == nil || !m2[k].Equal(v) { + return false + } + } + return true +} + func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { log.Info("filtering issues for feature: ", featureName) objects := make([]ObjectInfo, 0) link := "" // for oracle we shouldn't display any line for links + var minVersionsFixedIn map[string]*ybversion.YBVersion + var minVersionsFixedInSet bool + for _, issue := range schemaAnalysisReport.Issues { if strings.Contains(issue.Reason, issueReason) { objectInfo := ObjectInfo{ @@ -924,9 +951,16 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea } link = issue.DocsLink objects = append(objects, objectInfo) + if !minVersionsFixedInSet { + minVersionsFixedIn = issue.MinimumVersionsFixedIn + minVersionsFixedInSet = true + } + if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { + utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", featureName, minVersionsFixedIn, issue.MinimumVersionsFixedIn) + } } } - return UnsupportedFeature{featureName, objects, displayDDLInHTML, link, description} + return UnsupportedFeature{featureName, objects, displayDDLInHTML, link, description, minVersionsFixedIn} } func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { @@ -964,6 +998,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem } func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.SchemaReport, unsupportedIndexDatatypes []string) UnsupportedFeature { + // TODO: include MinimumVersionsFixedIn indexesOnComplexTypesFeature := UnsupportedFeature{ FeatureName: "Index on complex datatypes", DisplayDDL: false, @@ -1035,10 +1070,10 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { } unsupportedFeatures := make([]UnsupportedFeature, 0) - unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{UNSUPPORTED_INDEXES_FEATURE, unsupportedIndexes, false, "", ""}) - unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{VIRTUAL_COLUMNS_FEATURE, virtualColumns, false, "", ""}) - unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{INHERITED_TYPES_FEATURE, inheritedTypes, false, "", ""}) - unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{UNSUPPORTED_PARTITIONING_METHODS_FEATURE, unsupportedPartitionTypes, false, "", ""}) + unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{UNSUPPORTED_INDEXES_FEATURE, unsupportedIndexes, false, "", "", nil}) + unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{VIRTUAL_COLUMNS_FEATURE, virtualColumns, false, "", "", nil}) + unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{INHERITED_TYPES_FEATURE, inheritedTypes, false, "", "", nil}) + unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{UNSUPPORTED_PARTITIONING_METHODS_FEATURE, unsupportedPartitionTypes, false, "", "", nil}) return unsupportedFeatures, nil } @@ -1057,12 +1092,22 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U for reason, issues := range groupPlpgsqlIssuesByReason { var objects []ObjectInfo var docsLink string + var minVersionsFixedIn map[string]*ybversion.YBVersion + var minVersionsFixedInSet bool + for _, issue := range issues { objects = append(objects, ObjectInfo{ ObjectType: issue.ObjectType, ObjectName: issue.ObjectName, SqlStatement: issue.SqlStatement, }) + if !minVersionsFixedInSet { + minVersionsFixedIn = issue.MinimumVersionsFixedIn + minVersionsFixedInSet = true + } + if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { + utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", reason, minVersionsFixedIn, issue.MinimumVersionsFixedIn) + } docsLink = issue.DocsLink } feature := UnsupportedFeature{ @@ -1121,9 +1166,10 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error for _, issue := range issues { uqc := utils.UnsupportedQueryConstruct{ - Query: issue.SqlStatement, - ConstructTypeName: issue.TypeName, - DocsLink: issue.DocsLink, + Query: issue.SqlStatement, + ConstructTypeName: issue.TypeName, + DocsLink: issue.DocsLink, + MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn, } result = append(result, uqc) } @@ -1272,14 +1318,14 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration for _, col := range unsupportedDataTypesForLiveMigration { columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE}) + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) } if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { columns := make([]ObjectInfo, 0) for _, col := range unsupportedDataTypesForLiveMigrationWithFForFB { columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE}) + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) } assessmentReport.MigrationCaveats = migrationCaveats } @@ -1344,6 +1390,7 @@ func generateAssessmentReportHtml(reportDir string) error { "numKeysInMapStringObjectInfo": numKeysInMapStringObjectInfo, "groupByObjectName": groupByObjectName, "totalUniqueObjectNamesOfAllTypes": totalUniqueObjectNamesOfAllTypes, + "getSupportedVersionString": getSupportedVersionString, } tmpl := template.Must(template.New("report").Funcs(funcMap).Parse(string(bytesTemplate))) @@ -1389,6 +1436,20 @@ func split(value string, delimiter string) []string { return strings.Split(value, delimiter) } +func getSupportedVersionString(minimumVersionsFixedIn map[string]*ybversion.YBVersion) string { + if minimumVersionsFixedIn == nil { + return "" + } + supportedVersions := []string{} + for series, minVersionFixedIn := range minimumVersionsFixedIn { + if minVersionFixedIn == nil { + continue + } + supportedVersions = append(supportedVersions, fmt.Sprintf(">=%s (%s series)", minVersionFixedIn.String(), series)) + } + return strings.Join(supportedVersions, ", ") +} + func validateSourceDBTypeForAssessMigration() { if source.DBType == "" { utils.ErrExit("Error: required flag \"source-db-type\" not set") diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 0418797e2e..3e608da7c7 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1164,6 +1164,7 @@ func getMigrationComplexityForOracle(schemaDirectory string) (string, error) { // TODO: consider merging all unsupported field with single AssessmentReport struct member as AssessmentIssue type AssessmentReport struct { VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` MigrationComplexity string `json:"MigrationComplexity"` SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` @@ -1179,11 +1180,12 @@ type AssessmentReport struct { } type UnsupportedFeature struct { - FeatureName string `json:"FeatureName"` - Objects []ObjectInfo `json:"Objects"` - DisplayDDL bool `json:"-"` // just used by html format to display the DDL for some feature and object names for other - DocsLink string `json:"DocsLink,omitempty"` - FeatureDescription string `json:"FeatureDescription,omitempty"` + FeatureName string `json:"FeatureName"` + Objects []ObjectInfo `json:"Objects"` + DisplayDDL bool `json:"-"` // just used by html format to display the DDL for some feature and object names for other + DocsLink string `json:"DocsLink,omitempty"` + FeatureDescription string `json:"FeatureDescription,omitempty"` + MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"` // key: series (2024.1, 2.21, etc) } type ObjectInfo struct { diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index 69fdc9be46..74214c1d9b 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -411,7 +411,7 @@ func createTargetSchemas(conn *pgx.Conn) { schemaAnalysisReport := analyzeSchemaInternal( &srcdb.Source{ DBType: sourceDBType, - }) + }, false) switch sourceDBType { case "postgresql": // in case of postgreSQL as source, there can be multiple schemas present in a database diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index c172979362..85a10b1446 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -88,6 +88,8 @@

    Database Version: {{.}}

    {{end}} +

    Target YB Version: {{.TargetDBVersion}}

    + {{if eq .MigrationComplexity "NOT AVAILABLE"}} {{else}} @@ -198,6 +200,10 @@ {{ $hasUnsupportedFeatures = true }} {{if .DisplayDDL }}

    {{.FeatureName}}

    + {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +

    Supported in Versions: {{ $supporterVerStr }}

    + {{ end }}
      {{range .Objects}} @@ -207,6 +213,10 @@
    {{else}}

    {{.FeatureName}}

    + {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +

    Supported in Versions: {{ $supporterVerStr }}

    + {{ end }}
      {{range .Objects}} @@ -248,15 +258,18 @@ {{ $currentType := "" }} {{ $docsLink := "" }} + {{ $supporterVerStr := "" }} {{ range $i, $construct := .UnsupportedQueryConstructs }} {{ if ne $construct.ConstructTypeName $currentType }} {{ if ne $currentType "" }}
    - + {{ if $supporterVerStr }} + Supported in Versions: {{ $supporterVerStr }} + {{ end }} {{ if $docsLink }} - Link + Docs Link {{ else }} N/A {{ end }} @@ -264,6 +277,7 @@ {{ end }} {{ $docsLink = $construct.DocsLink }} + {{ $supporterVerStr = getSupportedVersionString $construct.MinimumVersionsFixedIn }} {{ $construct.ConstructTypeName }} @@ -279,8 +293,11 @@
    + {{ if $supporterVerStr }} + Supported in Versions: {{ $supporterVerStr }}
    + {{ end }} {{ if $docsLink }} - Link + Docs Link {{ else }} Not Available {{ end }} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.html b/yb-voyager/cmd/templates/schema_analysis_report.html index 00ee63d3be..34a4b6e230 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.html +++ b/yb-voyager/cmd/templates/schema_analysis_report.html @@ -63,6 +63,7 @@

    Schema Analysis Report

    Migration Information

    + {{if .SchemaSummary.SchemaNames }} @@ -121,6 +122,10 @@

    Issue in Object {{ $issue.ObjectType }}

  • Reason: {{ $issue.Reason }}
  • SQL Statement: 
    {{ $issue.SqlStatement }}
  • File Path: {{ $issue.FilePath }} [Preview]
    • + {{ $supporterVerStr := getSupportedVersionString $issue.MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +
  • Fixed in Versions: {{ $supporterVerStr }}
    • + {{ end }} {{ if $issue.Suggestion }}
  • Suggestion: {{ $issue.Suggestion }}
    • {{ end }} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.txt b/yb-voyager/cmd/templates/schema_analysis_report.txt index f031f83efd..2c1bcfffa8 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.txt +++ b/yb-voyager/cmd/templates/schema_analysis_report.txt @@ -9,6 +9,7 @@ Voyager Version : {{ .VoyagerVersion }} Database Name : {{ .SchemaSummary.DBName }} Schema Name(s) : {{ join .SchemaSummary.SchemaNames ", " }} DB Version : {{ .SchemaSummary.DBVersion }} +Target DB Version : {{ .TargetDBVersion }} {{if eq .MigrationComplexity "NOT AVAILABLE"}} {{else}} Migration Complexity : {{ .MigrationComplexity }} @@ -38,6 +39,10 @@ Issues - Suggestion : {{ .Suggestion }} {{ end }}{{ if .GH }} - Github Issue : {{ .GH }}{{ end }}{{ if .DocsLink }} - Docs Link : {{ .DocsLink }}{{ end }} +{{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} +{{ if $supporterVerStr }} + - Fixed in Versions: {{ $supporterVerStr }} +{{ end }} {{ end }}{{ else }} No issues found.{{ end }} diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index a7baef18a8..5f5dae6823 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -31,6 +31,9 @@ type Issue struct { } func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { + if i.MinimumVersionsFixedIn == nil { + return false, nil + } minVersionFixedInSeries, ok := i.MinimumVersionsFixedIn[v.Series()] if !ok { return false, nil diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index 4cb1e60474..c2b0abb8c9 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -19,6 +19,7 @@ import ( "sync" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) const ( @@ -73,10 +74,11 @@ var WaitChannel = make(chan int) // ================== Schema Report ============================== type SchemaReport struct { - VoyagerVersion string `json:"VoyagerVersion"` - MigrationComplexity string `json:"MigrationComplexity"` - SchemaSummary SchemaSummary `json:"Summary"` - Issues []Issue `json:"Issues"` + VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` + MigrationComplexity string `json:"MigrationComplexity"` + SchemaSummary SchemaSummary `json:"Summary"` + Issues []Issue `json:"Issues"` } type SchemaSummary struct { @@ -96,16 +98,29 @@ type DBObject struct { Details string `json:"Details,omitempty"` } +// TODO: support MinimumVersionsFixedIn in xml type Issue struct { - IssueType string `json:"IssueType"` - ObjectType string `json:"ObjectType"` - ObjectName string `json:"ObjectName"` - Reason string `json:"Reason"` - SqlStatement string `json:"SqlStatement,omitempty"` - FilePath string `json:"FilePath"` - Suggestion string `json:"Suggestion"` - GH string `json:"GH"` - DocsLink string `json:"DocsLink,omitempty"` + IssueType string `json:"IssueType"` + ObjectType string `json:"ObjectType"` + ObjectName string `json:"ObjectName"` + Reason string `json:"Reason"` + SqlStatement string `json:"SqlStatement,omitempty"` + FilePath string `json:"FilePath"` + Suggestion string `json:"Suggestion"` + GH string `json:"GH"` + DocsLink string `json:"DocsLink,omitempty"` + MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn" xml:"-"` // key: series (2024.1, 2.21, etc) +} + +func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { + if i.MinimumVersionsFixedIn == nil { + return false, nil + } + minVersionFixedInSeries, ok := i.MinimumVersionsFixedIn[v.Series()] + if !ok { + return false, nil + } + return v.GreaterThanOrEqual(minVersionFixedInSeries), nil } type IndexInfo struct { @@ -124,9 +139,10 @@ type TableColumnsDataTypes struct { } type UnsupportedQueryConstruct struct { - ConstructTypeName string - Query string - DocsLink string + ConstructTypeName string + Query string + DocsLink string + MinimumVersionsFixedIn map[string]*ybversion.YBVersion // key: series (2024.1, 2.21, etc) } // ================== Segment ============================== diff --git a/yb-voyager/src/ybversion/constants.go b/yb-voyager/src/ybversion/constants.go index 1e6f4afecc..0858c58ce4 100644 --- a/yb-voyager/src/ybversion/constants.go +++ b/yb-voyager/src/ybversion/constants.go @@ -15,8 +15,6 @@ limitations under the License. */ package ybversion -import "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" - const ( SERIES_2_14 = "2.14" SERIES_2_18 = "2.18" @@ -34,7 +32,7 @@ func init() { var err error V2024_1_3_1, err = NewYBVersion("2024.1.3.1") if err != nil { - utils.ErrExit("could not create version 2024.1") + panic("could not create version 2024.1.3.1") } LatestStable = V2024_1_3_1 } diff --git a/yb-voyager/src/ybversion/yb_version.go b/yb-voyager/src/ybversion/yb_version.go index dc4aab5437..f2afdc660c 100644 --- a/yb-voyager/src/ybversion/yb_version.go +++ b/yb-voyager/src/ybversion/yb_version.go @@ -102,10 +102,26 @@ func (ybv *YBVersion) GreaterThanOrEqual(other *YBVersion) bool { return ybv.Version.GreaterThanOrEqual(other.Version) } +func (ybv *YBVersion) Equal(other *YBVersion) bool { + return ybv.Version.Equal(other.Version) +} + func (ybv *YBVersion) String() string { return ybv.Original() } +// override the UnmarshalText method of Version. +// UnmarshalText implements encoding.TextUnmarshaler interface. +func (ybv *YBVersion) UnmarshalText(b []byte) error { + temp, err := NewYBVersion(string(b)) + if err != nil { + return err + } + + *ybv = *temp + return nil +} + func joinIntsWith(ints []int, delimiter string) string { strs := make([]string, len(ints)) for i, v := range ints { diff --git a/yb-voyager/src/ybversion/yb_version_test.go b/yb-voyager/src/ybversion/yb_version_test.go index dec6300480..b29711bad9 100644 --- a/yb-voyager/src/ybversion/yb_version_test.go +++ b/yb-voyager/src/ybversion/yb_version_test.go @@ -95,6 +95,9 @@ func TestLatestStable(t *testing.T) { response, err := http.Get(url) assert.NoErrorf(t, err, "could not access URL:%q", url) defer response.Body.Close() + if response.StatusCode == 403 { + t.Skip("skipping test; rate limit exceeded") + } assert.Equal(t, 200, response.StatusCode) body, err := io.ReadAll(response.Body) From 8b4874427a82c8189c35d8850800b1d3fa8e8637 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 5 Dec 2024 12:18:04 +0530 Subject: [PATCH 032/105] Implement ObjectCollector struct to collect objects names present in the provided node message (#2027) - Collect() method can be called for each node/msg while traversing the tree to collect all objects - Current coverage - extracts the table and function names(qualified) for all types of DMLs and some DDLs(completion todo) - Add unit test covering wide variety of cases where - SELECT present in CREATE TABLE, VIEW, COPY - Function name is present anywhere, nested function names also. --- yb-voyager/cmd/assessMigrationCommand.go | 56 +++- yb-voyager/src/queryissue/queryissue.go | 1 + .../queryissue/unsupported_dml_constructs.go | 7 +- .../unsupported_dml_constructs_test.go | 97 ++++++- yb-voyager/src/queryparser/helpers.go | 134 +++++----- .../src/queryparser/object_collector.go | 111 ++++++++ yb-voyager/src/queryparser/query_parser.go | 4 +- yb-voyager/src/queryparser/traversal.go | 26 ++ yb-voyager/src/queryparser/traversal_test.go | 245 ++++++++++++++++++ yb-voyager/src/srcdb/source.go | 4 + 10 files changed, 617 insertions(+), 68 deletions(-) create mode 100644 yb-voyager/src/queryparser/object_collector.go create mode 100644 yb-voyager/src/queryparser/traversal_test.go diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 9ffaad7243..b4cb76bb98 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -42,6 +42,7 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" "github.com/yugabyte/yb-voyager/yb-voyager/src/queryissue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" @@ -1158,6 +1159,19 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error for i := 0; i < len(executedQueries); i++ { query := executedQueries[i] log.Debugf("fetching unsupported query constructs for query - [%s]", query) + collectedSchemaList, err := queryparser.GetSchemaUsed(query) + if err != nil { // no need to error out if failed to get schemas for a query + log.Errorf("failed to get schemas used for query [%s]: %v", query, err) + continue + } + + log.Infof("collected schema list %v(len=%d) for query [%s]", collectedSchemaList, len(collectedSchemaList), query) + if !considerQueryForIssueDetection(collectedSchemaList) { + log.Infof("ignoring query due to difference in collected schema list %v(len=%d) vs source schema list %v(len=%d)", + collectedSchemaList, len(collectedSchemaList), source.GetSchemaList(), len(source.GetSchemaList())) + continue + } + issues, err := parserIssueDetector.GetDMLIssues(query, targetDbVersion) if err != nil { log.Errorf("failed while trying to fetch query issues in query - [%s]: %v", @@ -1173,7 +1187,6 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error } result = append(result, uqc) } - } // sort the slice to group same constructType in html and json reports @@ -1265,6 +1278,47 @@ func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []ut return unsupportedDataTypes, unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB, nil } +/* +Queries to ignore: +- Collected schemas is totally different than source schema list, not containing "" + +Queries to consider: +- Collected schemas subset of source schema list +- Collected schemas contains some from source schema list and some extras + +Caveats: +There can be a lot of false positives. +For example: standard sql functions like sum(), count() won't be qualified(pg_catalog) in queries generally. +Making the schema unknown for that object, resulting in query consider +*/ +func considerQueryForIssueDetection(collectedSchemaList []string) bool { + // filtering out pg_catalog schema, since it doesn't impact query consideration decision + collectedSchemaList = lo.Filter(collectedSchemaList, func(item string, _ int) bool { + return item != "pg_catalog" + }) + + sourceSchemaList := strings.Split(source.Schema, "|") + + // fallback in case: unable to collect objects or there are no object(s) in the query + if len(collectedSchemaList) == 0 { + return true + } + + // empty schemaname indicates presence of unqualified objectnames in query + if slices.Contains(collectedSchemaList, "") { + log.Debug("considering due to empty schema\n") + return true + } + + for _, collectedSchema := range collectedSchemaList { + if slices.Contains(sourceSchemaList, collectedSchema) { + log.Debugf("considering due to '%s' schema\n", collectedSchema) + return true + } + } + return false +} + const ( ORACLE_PARTITION_DEFAULT_COLOCATION = `For sharding/colocation recommendations, each partition is treated individually. During the export schema phase, all the partitions of a partitioned table are currently created as colocated by default. To manually modify the schema, please refer: https://github.com/yugabyte/yb-voyager/issues/1581.` diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 691a407ec6..1f4239f446 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -171,6 +171,7 @@ func (p *ParserIssueDetector) getDMLIssues(query string) ([]issue.IssueInstance, if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) } + var result []issue.IssueInstance var unsupportedConstructs []string visited := make(map[protoreflect.Message]bool) diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs.go b/yb-voyager/src/queryissue/unsupported_dml_constructs.go index 2c06c0734e..fb507e7c89 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs.go +++ b/yb-voyager/src/queryissue/unsupported_dml_constructs.go @@ -28,7 +28,8 @@ const ( XML_FUNCTIONS = "XML Functions" ) -// To Add a new unsupported query construct implement this interface +// To Add a new unsupported query construct implement this interface for all possible nodes for that construct +// each detector will work on specific type of node type UnsupportedConstructDetector interface { Detect(msg protoreflect.Message) ([]string, error) } @@ -58,7 +59,7 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) ([]string, error) { return nil, nil } - funcName := queryparser.GetFuncNameFromFuncCall(msg) + _, funcName := queryparser.GetFuncNameFromFuncCall(msg) log.Debugf("fetched function name from %s node: %q", queryparser.PG_QUERY_FUNCCALL_NODE, funcName) if constructType, isUnsupported := d.unsupportedFuncs[funcName]; isUnsupported { log.Debugf("detected unsupported function %q in msg - %+v", funcName, msg) @@ -88,7 +89,7 @@ func (d *ColumnRefDetector) Detect(msg protoreflect.Message) ([]string, error) { return nil, nil } - colName := queryparser.GetColNameFromColumnRef(msg) + _, colName := queryparser.GetColNameFromColumnRef(msg) log.Debugf("fetched column name from %s node: %q", queryparser.PG_QUERY_COLUMNREF_NODE, colName) if constructType, isUnsupported := d.unsupportedColumns[colName]; isUnsupported { log.Debugf("detected unsupported system column %q in msg - %+v", colName, msg) diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go b/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go index ba92994495..c2e0bc3724 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go +++ b/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go @@ -17,7 +17,6 @@ package queryissue import ( "fmt" - "sort" "testing" "github.com/samber/lo" @@ -561,9 +560,99 @@ RETURNING id, parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) + assert.ElementsMatch(t, expectedConstructs, unsupportedConstructs, "Detected constructs do not exactly match the expected constructs. Expected: %v, Actual: %v", expectedConstructs, unsupportedConstructs) + } +} + +func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { + tests := []struct { + Sql string + ExpectedObjects []string + ExpectedSchemas []string + }{ + { + Sql: `WITH LockedEmployees AS ( + SELECT *, pg_advisory_lock(xmin) AS lock_acquired + FROM public.employees + WHERE pg_try_advisory_lock(xmin) IS TRUE + ) + SELECT xmlelement(name "EmployeeData", xmlagg( + xmlelement(name "Employee", xmlattributes(id AS "ID"), + xmlforest(name AS "Name", xmin AS "TransactionID", xmax AS "ModifiedID")))) + FROM LockedEmployees + WHERE xmax IS NOT NULL;`, + /* + Limitation: limited coverage provided by objectCollector.Collect() right now. Might not detect some cases. + xmlelement, xmlforest etc are present under xml_expr node in parse tree not funccall node. + */ + ExpectedObjects: []string{"pg_advisory_lock", "public.employees", "pg_try_advisory_lock", "xmlagg", "lockedemployees"}, + ExpectedSchemas: []string{"public", ""}, + }, + { + Sql: `WITH Data AS ( + SELECT id, name, xmin, xmax, + pg_try_advisory_lock(id) AS lock_status, + xmlelement(name "info", xmlforest(name as "name", xmin as "transaction_start", xmax as "transaction_end")) as xml_info + FROM projects + WHERE xmin > 100 AND xmax < 500 + ) + SELECT x.id, x.xml_info + FROM Data x + WHERE x.lock_status IS TRUE;`, + ExpectedObjects: []string{"pg_try_advisory_lock", "projects", "data"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `UPDATE s1.employees + SET salary = salary * 1.1 + WHERE pg_try_advisory_xact_lock(ctid) IS TRUE AND department = 'Engineering' + RETURNING id, xmlelement(name "UpdatedEmployee", xmlattributes(id AS "ID"), + xmlforest(name AS "Name", salary AS "NewSalary", xmin AS "TransactionStartID", xmax AS "TransactionEndID"));`, + ExpectedObjects: []string{"s1.employees", "pg_try_advisory_xact_lock"}, + ExpectedSchemas: []string{"s1", ""}, + }, + } + + expectedConstructs := []string{ADVISORY_LOCKS, SYSTEM_COLUMNS, XML_FUNCTIONS} + + detectors := []UnsupportedConstructDetector{ + NewFuncCallDetector(), + NewColumnRefDetector(), + NewXmlExprDetector(), + } + for _, tc := range tests { + parseResult, err := queryparser.Parse(tc.Sql) + assert.NoError(t, err) + + visited := make(map[protoreflect.Message]bool) + unsupportedConstructs := []string{} + + objectCollector := queryparser.NewObjectCollector() + processor := func(msg protoreflect.Message) error { + for _, detector := range detectors { + log.Debugf("running detector %T", detector) + constructs, err := detector.Detect(msg) + if err != nil { + log.Debugf("error in detector %T: %v", detector, err) + return fmt.Errorf("error in detectors %T: %w", detector, err) + } + unsupportedConstructs = lo.Union(unsupportedConstructs, constructs) + } + objectCollector.Collect(msg) + return nil + } + + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) + err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + assert.ElementsMatch(t, expectedConstructs, unsupportedConstructs, "Detected constructs do not exactly match the expected constructs. Expected: %v, Actual: %v", expectedConstructs, unsupportedConstructs) + + collectedObjects := objectCollector.GetObjects() + collectedSchemas := objectCollector.GetSchemaList() - sort.Strings(unsupportedConstructs) - sort.Strings(expectedConstructs) - assert.Equal(t, expectedConstructs, unsupportedConstructs, "Detected constructs do not exactly match the expected constructs") + assert.ElementsMatch(t, tc.ExpectedObjects, collectedObjects, + "Objects list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedObjects, len(tc.ExpectedObjects), collectedObjects, len(collectedObjects)) + assert.ElementsMatch(t, tc.ExpectedSchemas, collectedSchemas, + "Schema list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedSchemas, len(tc.ExpectedSchemas), collectedSchemas, len(collectedSchemas)) } } diff --git a/yb-voyager/src/queryparser/helpers.go b/yb-voyager/src/queryparser/helpers.go index d0c1cb57a4..09c999c80b 100644 --- a/yb-voyager/src/queryparser/helpers.go +++ b/yb-voyager/src/queryparser/helpers.go @@ -30,17 +30,18 @@ const ( XML_FUNCTIONS_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#xml-functions-is-not-yet-supported" ) +func GetMsgFullName(msg protoreflect.Message) string { + return string(msg.Descriptor().FullName()) +} + // Sample example: {func_call:{funcname:{string:{sval:"pg_advisory_lock"}} -func GetFuncNameFromFuncCall(funcCallNode protoreflect.Message) string { +func GetFuncNameFromFuncCall(funcCallNode protoreflect.Message) (string, string) { if GetMsgFullName(funcCallNode) != PG_QUERY_FUNCCALL_NODE { - return "" + return "", "" } - funcnameField := funcCallNode.Get(funcCallNode.Descriptor().Fields().ByName("funcname")) - funcnameList := funcnameField.List() + funcnameList := GetListField(funcCallNode, "funcname") var names []string - - // TODO: simplification to directly access last item of funcnameList for i := 0; i < funcnameList.Len(); i++ { item := funcnameList.Get(i) name := GetStringValueFromNode(item.Message()) @@ -48,23 +49,23 @@ func GetFuncNameFromFuncCall(funcCallNode protoreflect.Message) string { names = append(names, name) } } + if len(names) == 0 { - return "" + return "", "" + } else if len(names) == 1 { + return "", names[0] } - return names[len(names)-1] // ignoring schema_name + return names[0], names[1] } // Sample example:: {column_ref:{fields:{string:{sval:"xmax"}} -func GetColNameFromColumnRef(columnRefNode protoreflect.Message) string { +func GetColNameFromColumnRef(columnRefNode protoreflect.Message) (string, string) { if GetMsgFullName(columnRefNode) != PG_QUERY_COLUMNREF_NODE { - return "" + return "", "" } - fields := columnRefNode.Get(columnRefNode.Descriptor().Fields().ByName("fields")) - fieldsList := fields.List() + fieldsList := GetListField(columnRefNode, "fields") var names []string - - // TODO: simplification to directly access last item of fieldsList for i := 0; i < fieldsList.Len(); i++ { item := fieldsList.Get(i) name := GetStringValueFromNode(item.Message()) @@ -72,10 +73,13 @@ func GetColNameFromColumnRef(columnRefNode protoreflect.Message) string { names = append(names, name) } } + if len(names) == 0 { - return "" + return "", "" + } else if len(names) == 1 { + return "", names[0] } - return names[len(names)-1] // ignoring schema_name + return names[0], names[1] } // Sample example:: {column_ref:{fields:{string:{sval:"s"}} fields:{string:{sval:"tableoid"}} location:7} @@ -100,13 +104,13 @@ func GetStringValueFromNode(nodeMsg protoreflect.Message) string { switch nodeType { // Represents a simple string literal in a query, such as names or values directly provided in the SQL text. case PG_QUERY_STRING_NODE: - return extractStringField(node, "sval") + return GetStringField(node, "sval") // Represents a constant value in SQL expressions, often used for literal values like strings, numbers, or keywords. case PG_QUERY_ACONST_NODE: - return extractAConstString(node) + return getStringFromAConstMsg(node) // Represents a type casting operation in SQL, where a value is explicitly converted from one type to another using a cast expression. case PG_QUERY_TYPECAST_NODE: - return traverseAndExtractAConst(node, "arg") + return getStringFromTypeCastMsg(node, "arg") // Represents the asterisk '*' used in SQL to denote the selection of all columns in a table. Example: SELECT * FROM employees; case PG_QUERY_ASTAR_NODE: return "" @@ -115,49 +119,22 @@ func GetStringValueFromNode(nodeMsg protoreflect.Message) string { } } -// extractStringField safely extracts a string field from a node -// Sample example:: {column_ref:{fields:{string:{sval:"s"}} fields:{string:{sval:"tableoid"}} location:7} -func extractStringField(node protoreflect.Message, fieldName string) string { - strField := node.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) - if strField == nil || !node.Has(strField) { - return "" - } - return node.Get(strField).String() -} - -// extractAConstString extracts the string from an 'A_Const' node's 'sval' field +// getStringFromAConstMsg extracts the string from an 'A_Const' node's 'sval' field // Sample example:: rowexpr:{a_const:{sval:{sval:"//Product"} location:124}} -func extractAConstString(aConstMsg protoreflect.Message) string { - // Extract the 'sval' field from 'A_Const' - svalField := aConstMsg.Descriptor().Fields().ByName("sval") - if svalField == nil || !aConstMsg.Has(svalField) { - return "" - } - - svalMsg := aConstMsg.Get(svalField).Message() - if svalMsg == nil || !svalMsg.IsValid() { - return "" - } - - // Ensure svalMsg is of type 'pg_query.String' - if svalMsg.Descriptor().FullName() != "pg_query.String" { +func getStringFromAConstMsg(aConstMsg protoreflect.Message) string { + svalMsg := GetMessageField(aConstMsg, "sval") + if svalMsg == nil { return "" } - // Extract the actual string value from 'pg_query.String' - return extractStringField(svalMsg, "sval") + return GetStringField(svalMsg, "sval") } -// traverseAndExtractAConst traverses to a specified field and extracts the 'A_Const' string value +// getStringFromTypeCastMsg traverses to a specified field and extracts the 'A_Const' string value // Sample example:: rowexpr:{type_cast:{arg:{a_const:{sval:{sval:"/order/item"} -func traverseAndExtractAConst(nodeMsg protoreflect.Message, fieldName string) string { - field := nodeMsg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) - if field == nil || !nodeMsg.Has(field) { - return "" - } - - childMsg := nodeMsg.Get(field).Message() - if childMsg == nil || !childMsg.IsValid() { +func getStringFromTypeCastMsg(nodeMsg protoreflect.Message, fieldName string) string { + childMsg := GetMessageField(nodeMsg, fieldName) + if childMsg == nil { return "" } @@ -281,10 +258,6 @@ func IsXPathExprForXmlTable(expression string) bool { return strings.HasPrefix(expression, "/") || strings.HasPrefix(expression, "//") } -func GetMsgFullName(msg protoreflect.Message) string { - return string(msg.Descriptor().FullName()) -} - // IsParameterPlaceholder checks if the given node represents a parameter placeholder like $1 func IsParameterPlaceholder(nodeMsg protoreflect.Message) bool { if nodeMsg == nil || !nodeMsg.IsValid() { @@ -334,3 +307,46 @@ func getOneofActiveField(msg protoreflect.Message, oneofName string) protoreflec // Determine which field within the oneof is set return msg.WhichOneof(oneofDescriptor) } + +// == Generic helper functions == + +// GetStringField retrieves a string field from a message. +// Sample example:: {column_ref:{fields:{string:{sval:"s"}} fields:{string:{sval:"tableoid"}} location:7} +func GetStringField(msg protoreflect.Message, fieldName string) string { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).String() + } + return "" +} + +// GetMessageField retrieves a message field from a message. +func GetMessageField(msg protoreflect.Message, fieldName string) protoreflect.Message { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).Message() + } + return nil +} + +// GetListField retrieves a list field from a message. +func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).List() + } + return nil +} + +// GetSchemaAndObjectName extracts the schema and object name from a list. +func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { + var schemaName, objectName string + + if nameList.Len() == 1 { + objectName = GetStringField(nameList.Get(0).Message(), "string") + } else if nameList.Len() == 2 { + schemaName = GetStringField(nameList.Get(0).Message(), "string") + objectName = GetStringField(nameList.Get(1).Message(), "string") + } + return schemaName, objectName +} diff --git a/yb-voyager/src/queryparser/object_collector.go b/yb-voyager/src/queryparser/object_collector.go new file mode 100644 index 0000000000..62f69c9c34 --- /dev/null +++ b/yb-voyager/src/queryparser/object_collector.go @@ -0,0 +1,111 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryparser + +import ( + "strings" + + "github.com/samber/lo" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// ObjectCollector collects unique schema-qualified object names. +type ObjectCollector struct { + objectSet map[string]bool +} + +func NewObjectCollector() *ObjectCollector { + return &ObjectCollector{ + objectSet: make(map[string]bool), + } +} + +/* +Collect processes a given node and extracts object names based on node type. +Cases covered: +1. [DML] SELECT queries - collect table/function object in it +2. [DML]Insert/Update/Delete queries +3. TODO: Coverage for DDLs (right now it worked with some cases like CREATE VIEW and CREATE SEQUENCE, but need to ensure all cases are covered) + +Collect() should be called from TraverseParseTree() to get all the objects in the parse tree of a stmt +*/ +func (c *ObjectCollector) Collect(msg protoreflect.Message) { + if msg == nil || !msg.IsValid() { + return + } + + nodeType := GetMsgFullName(msg) + switch nodeType { + // Extract table or view names in FROM clauses + case PG_QUERY_RANGEVAR_NODE: + schemaName := GetStringField(msg, "schemaname") + relName := GetStringField(msg, "relname") + objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName) + log.Debugf("[RangeVar] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) + c.addObject(objectName) + + // Extract target table names from DML statements + case PG_QUERY_INSERTSTMT_NODE, PG_QUERY_UPDATESTMT_NODE, PG_QUERY_DELETESTMT_NODE: + relationMsg := GetMessageField(msg, "relation") + if relationMsg != nil { + schemaName := GetStringField(relationMsg, "schemaname") + relName := GetStringField(relationMsg, "relname") + objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName) + log.Debugf("[IUD] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) + c.addObject(objectName) + } + + // Extract function names + case PG_QUERY_FUNCCALL_NODE: + schemaName, functionName := GetFuncNameFromFuncCall(msg) + if functionName == "" { + return + } + + objectName := lo.Ternary(schemaName != "", schemaName+"."+functionName, functionName) + log.Debugf("[Funccall] fetched schemaname=%s objectname=%s field\n", schemaName, objectName) + c.addObject(objectName) + + // Add more cases as needed for other message types + } +} + +// addObject adds an object name to the collector if it's not already present. +func (c *ObjectCollector) addObject(objectName string) { + if _, exists := c.objectSet[objectName]; !exists { + log.Debugf("adding object to object collector set: %s", objectName) + c.objectSet[objectName] = true + } +} + +// GetObjects returns a slice of collected unique object names. +func (c *ObjectCollector) GetObjects() []string { + return lo.Keys(c.objectSet) +} + +func (c *ObjectCollector) GetSchemaList() []string { + var schemaList []string + for obj := range c.objectSet { + splits := strings.Split(obj, ".") + if len(splits) == 1 { + schemaList = append(schemaList, "") + } else if len(splits) == 2 { + schemaList = append(schemaList, splits[0]) + } + } + return lo.Uniq(schemaList) +} diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/queryparser/query_parser.go index 2a4737cd7a..e8392092b9 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/queryparser/query_parser.go @@ -31,11 +31,13 @@ func Parse(query string) (*pg_query.ParseResult, error) { if err != nil { return nil, err } + log.Debugf("query: %s\n", query) + log.Debugf("parse tree: %v\n", tree) return tree, nil } func ParsePLPGSQLToJson(query string) (string, error) { - log.Debugf("parsing the PLPGSQL to json query-%s", query) + log.Debugf("parsing the PLPGSQL to json query [%s]", query) jsonString, err := pg_query.ParsePlPgSqlToJSON(query) if err != nil { return "", err diff --git a/yb-voyager/src/queryparser/traversal.go b/yb-voyager/src/queryparser/traversal.go index 1150e8f1b3..8f16517988 100644 --- a/yb-voyager/src/queryparser/traversal.go +++ b/yb-voyager/src/queryparser/traversal.go @@ -32,8 +32,14 @@ const ( PG_QUERY_XMLEXPR_NODE = "pg_query.XmlExpr" PG_QUERY_FUNCCALL_NODE = "pg_query.FuncCall" PG_QUERY_COLUMNREF_NODE = "pg_query.ColumnRef" + PG_QUERY_RANGEFUNCTION_NODE = "pg_query.RangeFunction" + PG_QUERY_RANGEVAR_NODE = "pg_query.RangeVar" PG_QUERY_RANGETABLEFUNC_NODE = "pg_query.RangeTableFunc" PG_QUERY_PARAMREF_NODE = "pg_query.ParamRef" + + PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" + PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" + PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" ) // function type for processing nodes during traversal @@ -204,3 +210,23 @@ func IsScalarKind(kind protoreflect.Kind) bool { protoreflect.BoolKind, protoreflect.StringKind, protoreflect.BytesKind, protoreflect.EnumKind} return slices.Contains(listOfScalarKinds, kind) } + +func GetSchemaUsed(query string) ([]string, error) { + parseTree, err := Parse(query) + if err != nil { + return nil, fmt.Errorf("error parsing query: %w", err) + } + + msg := GetProtoMessageFromParseTree(parseTree) + visited := make(map[protoreflect.Message]bool) + objectCollector := NewObjectCollector() + err = TraverseParseTree(msg, visited, func(msg protoreflect.Message) error { + objectCollector.Collect(msg) + return nil + }) + if err != nil { + return nil, fmt.Errorf("traversing parse tree: %w", err) + } + + return objectCollector.GetSchemaList(), nil +} diff --git a/yb-voyager/src/queryparser/traversal_test.go b/yb-voyager/src/queryparser/traversal_test.go new file mode 100644 index 0000000000..09f1dfd316 --- /dev/null +++ b/yb-voyager/src/queryparser/traversal_test.go @@ -0,0 +1,245 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryparser + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func TestObjectCollector(t *testing.T) { + tests := []struct { + Sql string + ExpectedObjects []string + ExpectedSchemas []string + }{ + { + Sql: `SELECT * from public.employees`, + ExpectedObjects: []string{"public.employees"}, + ExpectedSchemas: []string{"public"}, + }, + { + Sql: `SELECT * from employees`, + ExpectedObjects: []string{"employees"}, + ExpectedSchemas: []string{""}, // empty schemaname indicates unqualified objectname + }, + { + Sql: `SELECT * from s1.employees`, + ExpectedObjects: []string{"s1.employees"}, + ExpectedSchemas: []string{"s1"}, + }, + { + Sql: `SELECT * from s2.employees where salary > (Select salary from s3.employees)`, + ExpectedObjects: []string{"s3.employees", "s2.employees"}, + ExpectedSchemas: []string{"s3", "s2"}, + }, + { + Sql: `SELECT c.name, SUM(o.amount) AS total_spent FROM sales.customers c JOIN finance.orders o ON c.id = o.customer_id GROUP BY c.name HAVING SUM(o.amount) > 1000`, + ExpectedObjects: []string{"sales.customers", "finance.orders", "sum"}, + ExpectedSchemas: []string{"sales", "finance", ""}, + }, + { + Sql: `SELECT name FROM hr.employees WHERE department_id IN (SELECT id FROM public.departments WHERE location_id IN (SELECT id FROM eng.locations WHERE country = 'USA'))`, + ExpectedObjects: []string{"hr.employees", "public.departments", "eng.locations"}, + ExpectedSchemas: []string{"hr", "public", "eng"}, + }, + { + Sql: `SELECT name FROM sales.customers UNION SELECT name FROM marketing.customers;`, + ExpectedObjects: []string{"sales.customers", "marketing.customers"}, + ExpectedSchemas: []string{"sales", "marketing"}, + }, + { + Sql: `CREATE VIEW analytics.top_customers AS + SELECT user_id, COUNT(*) as order_count FROM public.orders GROUP BY user_id HAVING COUNT(*) > 10;`, + ExpectedObjects: []string{"analytics.top_customers", "public.orders", "count"}, + ExpectedSchemas: []string{"analytics", "public", ""}, + }, + { + Sql: `WITH user_orders AS ( + SELECT u.id, o.id as order_id + FROM users u + JOIN orders o ON u.id = o.user_id + WHERE o.amount > 100 + ), order_items AS ( + SELECT o.order_id, i.product_id + FROM order_items i + JOIN user_orders o ON i.order_id = o.order_id + ) + SELECT p.name, COUNT(oi.product_id) FROM products p + JOIN order_items oi ON p.id = oi.product_id GROUP BY p.name;`, + ExpectedObjects: []string{"users", "orders", "order_items", "user_orders", "products", "count"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `UPDATE finance.accounts + SET balance = balance + 1000 + WHERE account_id IN ( + SELECT account_id FROM public.users WHERE active = true + );`, + ExpectedObjects: []string{"finance.accounts", "public.users"}, + ExpectedSchemas: []string{"finance", "public"}, + }, + { + Sql: `SELECT classid, objid, refobjid FROM pg_depend WHERE refclassid = $1::regclass AND deptype = $2 ORDER BY 3`, + ExpectedObjects: []string{"pg_depend"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `SELECT pg_advisory_unlock_shared(100);`, + ExpectedObjects: []string{"pg_advisory_unlock_shared"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `SELECT xpath_exists('/employee/name', 'John'::xml)`, + ExpectedObjects: []string{"xpath_exists"}, + ExpectedSchemas: []string{""}, + }, + } + + for _, tc := range tests { + parseResult, err := Parse(tc.Sql) + assert.NoError(t, err) + + objectsCollector := NewObjectCollector() + processor := func(msg protoreflect.Message) error { + objectsCollector.Collect(msg) + return nil + } + + visited := make(map[protoreflect.Message]bool) + parseTreeMsg := GetProtoMessageFromParseTree(parseResult) + err = TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + + collectedObjects := objectsCollector.GetObjects() + collectedSchemas := objectsCollector.GetSchemaList() + + assert.ElementsMatch(t, tc.ExpectedObjects, collectedObjects, + "Objects list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedObjects, len(tc.ExpectedObjects), collectedObjects, len(collectedObjects)) + assert.ElementsMatch(t, tc.ExpectedSchemas, collectedSchemas, + "Schema list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedSchemas, len(tc.ExpectedSchemas), collectedSchemas, len(collectedSchemas)) + } +} + +// test focussed on collecting function names from DMLs +func TestObjectCollector2(t *testing.T) { + tests := []struct { + Sql string + ExpectedObjects []string + ExpectedSchemas []string + }{ + { + Sql: `SELECT finance.calculate_tax(amount) AS tax, name FROM sales.transactions;`, + ExpectedObjects: []string{"finance.calculate_tax", "sales.transactions"}, + ExpectedSchemas: []string{"finance", "sales"}, + }, + { + Sql: `SELECT hr.get_employee_details(e.id) FROM hr.employees e;`, + ExpectedObjects: []string{"hr.get_employee_details", "hr.employees"}, + ExpectedSchemas: []string{"hr"}, + }, + { + Sql: `Select now();`, + ExpectedObjects: []string{"now"}, + ExpectedSchemas: []string{""}, + }, + { // nested functions + Sql: `SELECT finance.calculate_bonus(sum(salary)) FROM hr.employees;`, + ExpectedObjects: []string{"finance.calculate_bonus", "sum", "hr.employees"}, + ExpectedSchemas: []string{"finance", "", "hr"}, + }, + { // functions as arguments in expressions + Sql: `SELECT e.name, CASE + WHEN e.salary > finance.calculate_bonus(e.salary) THEN 'High' + ELSE 'Low' + END AS salary_grade + FROM hr.employees e;`, + ExpectedObjects: []string{"finance.calculate_bonus", "hr.employees"}, + ExpectedSchemas: []string{"finance", "hr"}, + }, + { + Sql: `CREATE SEQUENCE finance.invoice_seq START 1000;`, + ExpectedObjects: []string{"finance.invoice_seq"}, + ExpectedSchemas: []string{"finance"}, + }, + { + Sql: `SELECT department, + MAX(CASE WHEN month = 'January' THEN sales ELSE 0 END) AS January_Sales, + MAX(CASE WHEN month = 'February' THEN sales ELSE 0 END) AS February_Sales + FROM sales_data + GROUP BY department;`, + ExpectedObjects: []string{"max", "sales_data"}, + ExpectedSchemas: []string{""}, + }, + { // quoted mixed case + Sql: `SELECT * FROM "SALES_DATA"."Order_Details";`, + ExpectedObjects: []string{"SALES_DATA.Order_Details"}, + ExpectedSchemas: []string{"SALES_DATA"}, + }, + { + Sql: `SELECT * FROM myfunc(analytics.calculate_metrics(2024)) AS cm(metrics);`, + ExpectedObjects: []string{"myfunc", "analytics.calculate_metrics"}, + ExpectedSchemas: []string{"", "analytics"}, + }, + { + Sql: `COPY (SELECT id, xmlagg(xmlparse(document xml_column)) AS combined_xml + FROM my_table + GROUP BY id) + TO '/path/to/file.csv' WITH CSV;`, + ExpectedObjects: []string{"xmlagg", "my_table"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `COPY (SELECT ctid, xmin, id, data FROM schema_name.my_table) + TO '/path/to/file_with_system_columns.csv' WITH CSV;`, + ExpectedObjects: []string{"schema_name.my_table"}, + ExpectedSchemas: []string{"schema_name"}, + }, + } + + for _, tc := range tests { + parseResult, err := Parse(tc.Sql) + assert.NoError(t, err) + + objectsCollector := NewObjectCollector() + processor := func(msg protoreflect.Message) error { + objectsCollector.Collect(msg) + return nil + } + + visited := make(map[protoreflect.Message]bool) + parseTreeMsg := GetProtoMessageFromParseTree(parseResult) + err = TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + + collectedObjects := objectsCollector.GetObjects() + collectedSchemas := objectsCollector.GetSchemaList() + + assert.ElementsMatch(t, tc.ExpectedObjects, collectedObjects, + "Objects list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedObjects, len(tc.ExpectedObjects), collectedObjects, len(collectedObjects)) + assert.ElementsMatch(t, tc.ExpectedSchemas, collectedSchemas, + "Schema list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedSchemas, len(tc.ExpectedSchemas), collectedSchemas, len(collectedSchemas)) + } +} + +/* + + + COPY (SELECT ctid, xmin, id, data FROM schema_name.my_table) + TO '/path/to/file_with_system_columns.csv' WITH CSV; +*/ diff --git a/yb-voyager/src/srcdb/source.go b/yb-voyager/src/srcdb/source.go index 945f2fbf77..605536c9fb 100644 --- a/yb-voyager/src/srcdb/source.go +++ b/yb-voyager/src/srcdb/source.go @@ -83,6 +83,10 @@ func (s *Source) GetOracleHome() string { } } +func (s *Source) GetSchemaList() []string { + return strings.Split(s.Schema, "|") +} + func (s *Source) IsOracleCDBSetup() bool { return (s.CDBName != "" || s.CDBTNSAlias != "" || s.CDBSid != "") } From 9994824af2bd87033d4fd4ec97c4d0295c276bf1 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 5 Dec 2024 16:19:04 +0530 Subject: [PATCH 033/105] Introduced DDLProcessor and DDLIssueDetector for parsing DDLs and detecting their issues. (#2020) Introduced DDLProcessor and DDLIssueDetector interfaces. DDLProcessor - Interface for each type of DDL to process it and store the information required to detect the issue in the DDLObject of that DDL, example TableProcessor - for parsing the CREATE TABLE DDL and storing the information related to this DDL in Table object such as isUnlogged, IsInherited , Columns , Constraints etc.. DDLIssueDetector - Interface for detecting the issues on the particular type of DDL by reading the DDLObject for required information for issue, example TableIssueDetector - detects issues on the CREATE TABLE for the issues like - Inherited table, unlogged table, unsupported datatypes etc.. Moved all the parser cases from checkStmtUsingParser() to the queryissue. --- .../tests/analyze-schema/expected_issues.json | 14 +- .../expectedAssessmentReport.json | 14 +- .../expected_schema_analysis_report.json | 14 +- .../expectedAssessmentReport.json | 4 +- .../expected_schema_analysis_report.json | 4 +- .../expectedAssessmentReport.json | 176 +-- .../expected_schema_analysis_report.json | 176 +-- yb-voyager/cmd/analyzeSchema.go | 1185 +---------------- yb-voyager/cmd/assessMigrationCommand.go | 11 +- yb-voyager/src/issue/constants.go | 48 +- yb-voyager/src/issue/ddl.go | 416 ++++++ yb-voyager/src/queryissue/helpers.go | 40 + yb-voyager/src/queryissue/queryissue.go | 189 ++- yb-voyager/src/queryissue/unsupported_ddls.go | 579 ++++++++ yb-voyager/src/queryparser/ddl_processor.go | 880 ++++++++++++ .../{helpers.go => helpers_protomsg.go} | 15 + yb-voyager/src/queryparser/helpers_struct.go | 290 ++++ yb-voyager/src/queryparser/query_parser.go | 198 +-- 18 files changed, 2698 insertions(+), 1555 deletions(-) create mode 100644 yb-voyager/src/queryissue/unsupported_ddls.go create mode 100644 yb-voyager/src/queryparser/ddl_processor.go rename yb-voyager/src/queryparser/{helpers.go => helpers_protomsg.go} (97%) create mode 100644 yb-voyager/src/queryparser/helpers_struct.go diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index af94b1944e..b836663537 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -287,9 +287,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "abc", + "ObjectName": "abc ON public.example", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX abc ON public.example USING btree (new_id) WITH (fillfactor='70'); ", + "SqlStatement": "CREATE INDEX abc ON public.example USING btree (new_id) WITH (fillfactor='70');", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -298,9 +298,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "abc", + "ObjectName": "abc ON schema2.example", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX abc ON schema2.example USING btree (new_id) WITH (fillfactor='70'); ", + "SqlStatement": "CREATE INDEX abc ON schema2.example USING btree (new_id) WITH (fillfactor='70');", "Suggestion": "Remove the storage parameters from the DDL", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -1117,7 +1117,7 @@ "Reason": "DEFERRABLE constraints not supported yet", "SqlStatement": "create table unique_def_test(id int UNIQUE DEFERRABLE, c1 int);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", - "Suggestion": "Remove these constraints from the exported schema and make the necessary changes to the application before pointing it to target", + "Suggestion": "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1709", "MinimumVersionsFixedIn": null }, @@ -1192,7 +1192,7 @@ "ObjectType": "FOREIGN TABLE", "ObjectName": "public.locations", "Reason": "Foreign tables require manual intervention.", - "SqlStatement": "CREATE FOREIGN TABLE public.locations ( id integer NOT NULL, name character varying(100), geom geometry(Point,4326) ) SERVER remote_server OPTIONS ( schema_name 'public', table_name 'remote_locations' ); ", + "SqlStatement": "CREATE FOREIGN TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326)\n ) SERVER remote_server\nOPTIONS (\n schema_name 'public',\n table_name 'remote_locations'\n);", "Suggestion": "SERVER 'remote_server', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", @@ -1510,7 +1510,7 @@ "ObjectType": "FOREIGN TABLE", "ObjectName": "tbl_p", "Reason": "Foreign tables require manual intervention.", - "SqlStatement": "CREATE FOREIGN TABLE tbl_p( \tid int PRIMARY KEY ) SERVER remote_server OPTIONS ( schema_name 'public', table_name 'remote_table' ); ", + "SqlStatement": "CREATE FOREIGN TABLE tbl_p(\n\tid int PRIMARY KEY\n) SERVER remote_server\nOPTIONS (\n schema_name 'public',\n table_name 'remote_table'\n);", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", "Suggestion": "SERVER 'remote_server', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 2b49cf1ed9..6c6a0be1e4 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -602,16 +602,16 @@ "FeatureName": "Storage parameters in DDLs", "Objects": [ { - "ObjectName": "gin_idx", - "SqlStatement": "CREATE INDEX gin_idx ON idx_ex.films USING gin (to_tsvector('english'::regconfig, title)) WITH (fastupdate=off); " + "ObjectName": "gin_idx ON idx_ex.films", + "SqlStatement": "CREATE INDEX gin_idx ON idx_ex.films USING gin (to_tsvector('english'::regconfig, title)) WITH (fastupdate=off);" }, { - "ObjectName": "title_idx", - "SqlStatement": "CREATE UNIQUE INDEX title_idx ON idx_ex.films USING btree (title) WITH (fillfactor='70'); " + "ObjectName": "title_idx ON idx_ex.films", + "SqlStatement": "CREATE UNIQUE INDEX title_idx ON idx_ex.films USING btree (title) WITH (fillfactor='70');" }, { - "ObjectName": "title_idx_with_duplicates", - "SqlStatement": "CREATE INDEX title_idx_with_duplicates ON idx_ex.films USING btree (title) WITH (deduplicate_items=off); " + "ObjectName": "title_idx_with_duplicates ON idx_ex.films", + "SqlStatement": "CREATE INDEX title_idx_with_duplicates ON idx_ex.films USING btree (title) WITH (deduplicate_items=off);" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", @@ -5539,7 +5539,7 @@ "Objects": [ { "ObjectName": "foreign_db_example.technically_doesnt_exist", - "SqlStatement": "CREATE FOREIGN TABLE foreign_db_example.technically_doesnt_exist ( id integer, uses_type foreign_db_example.example_type, _uses_type foreign_db_example.example_type GENERATED ALWAYS AS (uses_type) STORED, positive_number foreign_db_example.positive_number, _positive_number foreign_db_example.positive_number GENERATED ALWAYS AS (positive_number) STORED, CONSTRAINT imaginary_table_id_gt_1 CHECK ((id \u003e 1)) ) SERVER technically_this_server; " + "SqlStatement": "CREATE FOREIGN TABLE foreign_db_example.technically_doesnt_exist ( id integer, uses_type foreign_db_example.example_type, _uses_type foreign_db_example.example_type GENERATED ALWAYS AS (uses_type) STORED, positive_number foreign_db_example.positive_number, _positive_number foreign_db_example.positive_number GENERATED ALWAYS AS (positive_number) STORED, CONSTRAINT imaginary_table_id_gt_1 CHECK ((id \u003e 1)) ) SERVER technically_this_server;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index fd4eefb46d..c5a4f343ef 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -240,9 +240,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "gin_idx", + "ObjectName": "gin_idx ON idx_ex.films", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX gin_idx ON idx_ex.films USING gin (to_tsvector('english'::regconfig, title)) WITH (fastupdate=off); ", + "SqlStatement": "CREATE INDEX gin_idx ON idx_ex.films USING gin (to_tsvector('english'::regconfig, title)) WITH (fastupdate=off);", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -252,9 +252,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "title_idx", + "ObjectName": "title_idx ON idx_ex.films", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE UNIQUE INDEX title_idx ON idx_ex.films USING btree (title) WITH (fillfactor='70'); ", + "SqlStatement": "CREATE UNIQUE INDEX title_idx ON idx_ex.films USING btree (title) WITH (fillfactor='70');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -324,9 +324,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "title_idx_with_duplicates", + "ObjectName": "title_idx_with_duplicates ON idx_ex.films", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX title_idx_with_duplicates ON idx_ex.films USING btree (title) WITH (deduplicate_items=off); ", + "SqlStatement": "CREATE INDEX title_idx_with_duplicates ON idx_ex.films USING btree (title) WITH (deduplicate_items=off);", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -362,7 +362,7 @@ "ObjectType": "FOREIGN TABLE", "ObjectName": "foreign_db_example.technically_doesnt_exist", "Reason": "Foreign tables require manual intervention.", - "SqlStatement": "CREATE FOREIGN TABLE foreign_db_example.technically_doesnt_exist ( id integer, uses_type foreign_db_example.example_type, _uses_type foreign_db_example.example_type GENERATED ALWAYS AS (uses_type) STORED, positive_number foreign_db_example.positive_number, _positive_number foreign_db_example.positive_number GENERATED ALWAYS AS (positive_number) STORED, CONSTRAINT imaginary_table_id_gt_1 CHECK ((id \u003e 1)) ) SERVER technically_this_server; ", + "SqlStatement": "CREATE FOREIGN TABLE foreign_db_example.technically_doesnt_exist ( id integer, uses_type foreign_db_example.example_type, _uses_type foreign_db_example.example_type GENERATED ALWAYS AS (uses_type) STORED, positive_number foreign_db_example.positive_number, _positive_number foreign_db_example.positive_number GENERATED ALWAYS AS (positive_number) STORED, CONSTRAINT imaginary_table_id_gt_1 CHECK ((id \u003e 1)) ) SERVER technically_this_server;", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/foreign_table.sql", "Suggestion": "SERVER 'technically_this_server', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index 4670c68ce9..8c511ac3cc 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -230,11 +230,11 @@ "Objects": [ { "ObjectName": "public.f_c", - "SqlStatement": "CREATE FOREIGN TABLE public.f_c ( i integer NOT NULL, t integer, x text ) SERVER p10 OPTIONS ( table_name 'c' ); " + "SqlStatement": "CREATE FOREIGN TABLE public.f_c ( i integer NOT NULL, t integer, x text ) SERVER p10 OPTIONS ( table_name 'c' );" }, { "ObjectName": "public.f_t", - "SqlStatement": "CREATE FOREIGN TABLE public.f_t ( i integer NOT NULL, ts timestamp(0) with time zone DEFAULT now(), j json, t text, e public.myenum, c public.mycomposit ) SERVER p10 OPTIONS ( table_name 't' ); " + "SqlStatement": "CREATE FOREIGN TABLE public.f_t ( i integer NOT NULL, ts timestamp(0) with time zone DEFAULT now(), j json, t text, e public.myenum, c public.mycomposit ) SERVER p10 OPTIONS ( table_name 't' );" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", diff --git a/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json index 19ad430ca4..f907e1d6b2 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json @@ -65,7 +65,7 @@ "ObjectType": "FOREIGN TABLE", "ObjectName": "public.f_c", "Reason": "Foreign tables require manual intervention.", - "SqlStatement": "CREATE FOREIGN TABLE public.f_c ( i integer NOT NULL, t integer, x text ) SERVER p10 OPTIONS ( table_name 'c' ); ", + "SqlStatement": "CREATE FOREIGN TABLE public.f_c ( i integer NOT NULL, t integer, x text ) SERVER p10 OPTIONS ( table_name 'c' );", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/pgtbrus/export-dir/schema/tables/foreign_table.sql", "Suggestion": "SERVER 'p10', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", @@ -77,7 +77,7 @@ "ObjectType": "FOREIGN TABLE", "ObjectName": "public.f_t", "Reason": "Foreign tables require manual intervention.", - "SqlStatement": "CREATE FOREIGN TABLE public.f_t ( i integer NOT NULL, ts timestamp(0) with time zone DEFAULT now(), j json, t text, e public.myenum, c public.mycomposit ) SERVER p10 OPTIONS ( table_name 't' ); ", + "SqlStatement": "CREATE FOREIGN TABLE public.f_t ( i integer NOT NULL, ts timestamp(0) with time zone DEFAULT now(), j json, t text, e public.myenum, c public.mycomposit ) SERVER p10 OPTIONS ( table_name 't' );", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/pgtbrus/export-dir/schema/tables/foreign_table.sql", "Suggestion": "SERVER 'p10', and USER MAPPING should be created manually on the target to create and use the foreign table", "GH": "https://github.com/yugabyte/yb-voyager/issues/1627", diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index adaf24f5e8..f6be2c83ed 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -151,180 +151,180 @@ "FeatureName": "Storage parameters in DDLs", "Objects": [ { - "ObjectName": "badges_date_idx", - "SqlStatement": "CREATE INDEX badges_date_idx ON public.badges USING btree (date) WITH (fillfactor='100'); " + "ObjectName": "badges_date_idx ON public.badges", + "SqlStatement": "CREATE INDEX badges_date_idx ON public.badges USING btree (date) WITH (fillfactor='100');" }, { - "ObjectName": "badges_name_idx", - "SqlStatement": "CREATE INDEX badges_name_idx ON public.badges USING btree (name) WITH (fillfactor='100'); " + "ObjectName": "badges_name_idx ON public.badges", + "SqlStatement": "CREATE INDEX badges_name_idx ON public.badges USING btree (name) WITH (fillfactor='100');" }, { - "ObjectName": "badges_user_id_idx", - "SqlStatement": "CREATE INDEX badges_user_id_idx ON public.badges USING btree (userid) WITH (fillfactor='100'); " + "ObjectName": "badges_user_id_idx ON public.badges", + "SqlStatement": "CREATE INDEX badges_user_id_idx ON public.badges USING btree (userid) WITH (fillfactor='100');" }, { - "ObjectName": "cmnts_creation_date_idx", - "SqlStatement": "CREATE INDEX cmnts_creation_date_idx ON public.comments USING btree (creationdate) WITH (fillfactor='100'); " + "ObjectName": "cmnts_creation_date_idx ON public.comments", + "SqlStatement": "CREATE INDEX cmnts_creation_date_idx ON public.comments USING btree (creationdate) WITH (fillfactor='100');" }, { - "ObjectName": "cmnts_postid_idx", - "SqlStatement": "CREATE INDEX cmnts_postid_idx ON public.comments USING hash (postid) WITH (fillfactor='100'); " + "ObjectName": "cmnts_postid_idx ON public.comments", + "SqlStatement": "CREATE INDEX cmnts_postid_idx ON public.comments USING hash (postid) WITH (fillfactor='100');" }, { - "ObjectName": "cmnts_score_idx", - "SqlStatement": "CREATE INDEX cmnts_score_idx ON public.comments USING btree (score) WITH (fillfactor='100'); " + "ObjectName": "cmnts_score_idx ON public.comments", + "SqlStatement": "CREATE INDEX cmnts_score_idx ON public.comments USING btree (score) WITH (fillfactor='100');" }, { - "ObjectName": "cmnts_userid_idx", - "SqlStatement": "CREATE INDEX cmnts_userid_idx ON public.comments USING btree (userid) WITH (fillfactor='100'); " + "ObjectName": "cmnts_userid_idx ON public.comments", + "SqlStatement": "CREATE INDEX cmnts_userid_idx ON public.comments USING btree (userid) WITH (fillfactor='100');" }, { - "ObjectName": "ph_creation_date_idx", - "SqlStatement": "CREATE INDEX ph_creation_date_idx ON public.posthistory USING btree (creationdate) WITH (fillfactor='100'); " + "ObjectName": "ph_creation_date_idx ON public.posthistory", + "SqlStatement": "CREATE INDEX ph_creation_date_idx ON public.posthistory USING btree (creationdate) WITH (fillfactor='100');" }, { - "ObjectName": "ph_post_type_id_idx", - "SqlStatement": "CREATE INDEX ph_post_type_id_idx ON public.posthistory USING btree (posthistorytypeid) WITH (fillfactor='100'); " + "ObjectName": "ph_post_type_id_idx ON public.posthistory", + "SqlStatement": "CREATE INDEX ph_post_type_id_idx ON public.posthistory USING btree (posthistorytypeid) WITH (fillfactor='100');" }, { - "ObjectName": "ph_postid_idx", - "SqlStatement": "CREATE INDEX ph_postid_idx ON public.posthistory USING hash (postid) WITH (fillfactor='100'); " + "ObjectName": "ph_postid_idx ON public.posthistory", + "SqlStatement": "CREATE INDEX ph_postid_idx ON public.posthistory USING hash (postid) WITH (fillfactor='100');" }, { - "ObjectName": "ph_revguid_idx", - "SqlStatement": "CREATE INDEX ph_revguid_idx ON public.posthistory USING btree (revisionguid) WITH (fillfactor='100'); " + "ObjectName": "ph_revguid_idx ON public.posthistory", + "SqlStatement": "CREATE INDEX ph_revguid_idx ON public.posthistory USING btree (revisionguid) WITH (fillfactor='100');" }, { - "ObjectName": "ph_userid_idx", - "SqlStatement": "CREATE INDEX ph_userid_idx ON public.posthistory USING btree (userid) WITH (fillfactor='100'); " + "ObjectName": "ph_userid_idx ON public.posthistory", + "SqlStatement": "CREATE INDEX ph_userid_idx ON public.posthistory USING btree (userid) WITH (fillfactor='100');" }, { - "ObjectName": "postlinks_post_id_idx", - "SqlStatement": "CREATE INDEX postlinks_post_id_idx ON public.postlinks USING btree (postid) WITH (fillfactor='100'); " + "ObjectName": "postlinks_post_id_idx ON public.postlinks", + "SqlStatement": "CREATE INDEX postlinks_post_id_idx ON public.postlinks USING btree (postid) WITH (fillfactor='100');" }, { - "ObjectName": "postlinks_related_post_id_idx", - "SqlStatement": "CREATE INDEX postlinks_related_post_id_idx ON public.postlinks USING btree (relatedpostid) WITH (fillfactor='100'); " + "ObjectName": "postlinks_related_post_id_idx ON public.postlinks", + "SqlStatement": "CREATE INDEX postlinks_related_post_id_idx ON public.postlinks USING btree (relatedpostid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_accepted_answer_id_idx", - "SqlStatement": "CREATE INDEX posts_accepted_answer_id_idx ON public.posts USING btree (acceptedanswerid) WITH (fillfactor='100'); " + "ObjectName": "posts_accepted_answer_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_accepted_answer_id_idx ON public.posts USING btree (acceptedanswerid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_answer_count_idx", - "SqlStatement": "CREATE INDEX posts_answer_count_idx ON public.posts USING btree (answercount) WITH (fillfactor='100'); " + "ObjectName": "posts_answer_count_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_answer_count_idx ON public.posts USING btree (answercount) WITH (fillfactor='100');" }, { - "ObjectName": "posts_comment_count_idx", - "SqlStatement": "CREATE INDEX posts_comment_count_idx ON public.posts USING btree (commentcount) WITH (fillfactor='100'); " + "ObjectName": "posts_comment_count_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_comment_count_idx ON public.posts USING btree (commentcount) WITH (fillfactor='100');" }, { - "ObjectName": "posts_creation_date_idx", - "SqlStatement": "CREATE INDEX posts_creation_date_idx ON public.posts USING btree (creationdate) WITH (fillfactor='100'); " + "ObjectName": "posts_creation_date_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_creation_date_idx ON public.posts USING btree (creationdate) WITH (fillfactor='100');" }, { - "ObjectName": "posts_favorite_count_idx", - "SqlStatement": "CREATE INDEX posts_favorite_count_idx ON public.posts USING btree (favoritecount) WITH (fillfactor='100'); " + "ObjectName": "posts_favorite_count_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_favorite_count_idx ON public.posts USING btree (favoritecount) WITH (fillfactor='100');" }, { - "ObjectName": "posts_id_accepted_answers_id_idx", - "SqlStatement": "CREATE INDEX posts_id_accepted_answers_id_idx ON public.posts USING btree (id, acceptedanswerid) WITH (fillfactor='100'); " + "ObjectName": "posts_id_accepted_answers_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_id_accepted_answers_id_idx ON public.posts USING btree (id, acceptedanswerid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_id_parent_id_idx", - "SqlStatement": "CREATE INDEX posts_id_parent_id_idx ON public.posts USING btree (id, parentid) WITH (fillfactor='100'); " + "ObjectName": "posts_id_parent_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_id_parent_id_idx ON public.posts USING btree (id, parentid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_id_post_type_id_idx", - "SqlStatement": "CREATE INDEX posts_id_post_type_id_idx ON public.posts USING btree (id, posttypeid) WITH (fillfactor='100'); " + "ObjectName": "posts_id_post_type_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_id_post_type_id_idx ON public.posts USING btree (id, posttypeid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_owner_user_id_creation_date_idx", - "SqlStatement": "CREATE INDEX posts_owner_user_id_creation_date_idx ON public.posts USING btree (owneruserid, creationdate) WITH (fillfactor='100'); " + "ObjectName": "posts_owner_user_id_creation_date_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_owner_user_id_creation_date_idx ON public.posts USING btree (owneruserid, creationdate) WITH (fillfactor='100');" }, { - "ObjectName": "posts_owner_user_id_idx", - "SqlStatement": "CREATE INDEX posts_owner_user_id_idx ON public.posts USING hash (owneruserid) WITH (fillfactor='100'); " + "ObjectName": "posts_owner_user_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_owner_user_id_idx ON public.posts USING hash (owneruserid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_parent_id_idx", - "SqlStatement": "CREATE INDEX posts_parent_id_idx ON public.posts USING btree (parentid) WITH (fillfactor='100'); " + "ObjectName": "posts_parent_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_parent_id_idx ON public.posts USING btree (parentid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_post_type_id_idx", - "SqlStatement": "CREATE INDEX posts_post_type_id_idx ON public.posts USING btree (posttypeid) WITH (fillfactor='100'); " + "ObjectName": "posts_post_type_id_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_post_type_id_idx ON public.posts USING btree (posttypeid) WITH (fillfactor='100');" }, { - "ObjectName": "posts_score_idx", - "SqlStatement": "CREATE INDEX posts_score_idx ON public.posts USING btree (score) WITH (fillfactor='100'); " + "ObjectName": "posts_score_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_score_idx ON public.posts USING btree (score) WITH (fillfactor='100');" }, { - "ObjectName": "posts_viewcount_idx", - "SqlStatement": "CREATE INDEX posts_viewcount_idx ON public.posts USING btree (viewcount) WITH (fillfactor='100'); " + "ObjectName": "posts_viewcount_idx ON public.posts", + "SqlStatement": "CREATE INDEX posts_viewcount_idx ON public.posts USING btree (viewcount) WITH (fillfactor='100');" }, { - "ObjectName": "posttags_postid_idx", - "SqlStatement": "CREATE INDEX posttags_postid_idx ON public.posttags USING hash (postid) WITH (fillfactor='100'); " + "ObjectName": "posttags_postid_idx ON public.posttags", + "SqlStatement": "CREATE INDEX posttags_postid_idx ON public.posttags USING hash (postid) WITH (fillfactor='100');" }, { - "ObjectName": "posttags_tagid_idx", - "SqlStatement": "CREATE INDEX posttags_tagid_idx ON public.posttags USING btree (tagid) WITH (fillfactor='100'); " + "ObjectName": "posttags_tagid_idx ON public.posttags", + "SqlStatement": "CREATE INDEX posttags_tagid_idx ON public.posttags USING btree (tagid) WITH (fillfactor='100');" }, { - "ObjectName": "tags_count_idx", - "SqlStatement": "CREATE INDEX tags_count_idx ON public.tags USING btree (count) WITH (fillfactor='100'); " + "ObjectName": "tags_count_idx ON public.tags", + "SqlStatement": "CREATE INDEX tags_count_idx ON public.tags USING btree (count) WITH (fillfactor='100');" }, { - "ObjectName": "tags_name_idx", - "SqlStatement": "CREATE INDEX tags_name_idx ON public.tags USING hash (tagname) WITH (fillfactor='100'); " + "ObjectName": "tags_name_idx ON public.tags", + "SqlStatement": "CREATE INDEX tags_name_idx ON public.tags USING hash (tagname) WITH (fillfactor='100');" }, { - "ObjectName": "user_acc_id_idx", - "SqlStatement": "CREATE INDEX user_acc_id_idx ON public.users USING hash (accountid) WITH (fillfactor='100'); " + "ObjectName": "user_acc_id_idx ON public.users", + "SqlStatement": "CREATE INDEX user_acc_id_idx ON public.users USING hash (accountid) WITH (fillfactor='100');" }, { - "ObjectName": "user_created_at_idx", - "SqlStatement": "CREATE INDEX user_created_at_idx ON public.users USING btree (creationdate) WITH (fillfactor='100'); " + "ObjectName": "user_created_at_idx ON public.users", + "SqlStatement": "CREATE INDEX user_created_at_idx ON public.users USING btree (creationdate) WITH (fillfactor='100');" }, { - "ObjectName": "user_display_idx", - "SqlStatement": "CREATE INDEX user_display_idx ON public.users USING hash (displayname) WITH (fillfactor='100'); " + "ObjectName": "user_display_idx ON public.users", + "SqlStatement": "CREATE INDEX user_display_idx ON public.users USING hash (displayname) WITH (fillfactor='100');" }, { - "ObjectName": "user_down_votes_idx", - "SqlStatement": "CREATE INDEX user_down_votes_idx ON public.users USING btree (downvotes) WITH (fillfactor='100'); " + "ObjectName": "user_down_votes_idx ON public.users", + "SqlStatement": "CREATE INDEX user_down_votes_idx ON public.users USING btree (downvotes) WITH (fillfactor='100');" }, { - "ObjectName": "user_up_votes_idx", - "SqlStatement": "CREATE INDEX user_up_votes_idx ON public.users USING btree (upvotes) WITH (fillfactor='100'); " + "ObjectName": "user_up_votes_idx ON public.users", + "SqlStatement": "CREATE INDEX user_up_votes_idx ON public.users USING btree (upvotes) WITH (fillfactor='100');" }, { - "ObjectName": "usertagqa_all_qa_posts_idx", - "SqlStatement": "CREATE INDEX usertagqa_all_qa_posts_idx ON public.usertagqa USING btree (((questions + answers))) WITH (fillfactor='100'); " + "ObjectName": "usertagqa_all_qa_posts_idx ON public.usertagqa", + "SqlStatement": "CREATE INDEX usertagqa_all_qa_posts_idx ON public.usertagqa USING btree (((questions + answers))) WITH (fillfactor='100');" }, { - "ObjectName": "usertagqa_answers_idx", - "SqlStatement": "CREATE INDEX usertagqa_answers_idx ON public.usertagqa USING btree (answers) WITH (fillfactor='100'); " + "ObjectName": "usertagqa_answers_idx ON public.usertagqa", + "SqlStatement": "CREATE INDEX usertagqa_answers_idx ON public.usertagqa USING btree (answers) WITH (fillfactor='100');" }, { - "ObjectName": "usertagqa_questions_answers_idx", - "SqlStatement": "CREATE INDEX usertagqa_questions_answers_idx ON public.usertagqa USING btree (questions, answers) WITH (fillfactor='100'); " + "ObjectName": "usertagqa_questions_answers_idx ON public.usertagqa", + "SqlStatement": "CREATE INDEX usertagqa_questions_answers_idx ON public.usertagqa USING btree (questions, answers) WITH (fillfactor='100');" }, { - "ObjectName": "usertagqa_questions_idx", - "SqlStatement": "CREATE INDEX usertagqa_questions_idx ON public.usertagqa USING btree (questions) WITH (fillfactor='100'); " + "ObjectName": "usertagqa_questions_idx ON public.usertagqa", + "SqlStatement": "CREATE INDEX usertagqa_questions_idx ON public.usertagqa USING btree (questions) WITH (fillfactor='100');" }, { - "ObjectName": "votes_creation_date_idx", - "SqlStatement": "CREATE INDEX votes_creation_date_idx ON public.votes USING btree (creationdate) WITH (fillfactor='100'); " + "ObjectName": "votes_creation_date_idx ON public.votes", + "SqlStatement": "CREATE INDEX votes_creation_date_idx ON public.votes USING btree (creationdate) WITH (fillfactor='100');" }, { - "ObjectName": "votes_post_id_idx", - "SqlStatement": "CREATE INDEX votes_post_id_idx ON public.votes USING hash (postid) WITH (fillfactor='100'); " + "ObjectName": "votes_post_id_idx ON public.votes", + "SqlStatement": "CREATE INDEX votes_post_id_idx ON public.votes USING hash (postid) WITH (fillfactor='100');" }, { - "ObjectName": "votes_type_idx", - "SqlStatement": "CREATE INDEX votes_type_idx ON public.votes USING btree (votetypeid) WITH (fillfactor='100'); " + "ObjectName": "votes_type_idx ON public.votes", + "SqlStatement": "CREATE INDEX votes_type_idx ON public.votes USING btree (votetypeid) WITH (fillfactor='100');" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", diff --git a/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json index db788c7432..f40bc041d6 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json @@ -45,9 +45,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "badges_date_idx", + "ObjectName": "badges_date_idx ON public.badges", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX badges_date_idx ON public.badges USING btree (date) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX badges_date_idx ON public.badges USING btree (date) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -57,9 +57,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "badges_name_idx", + "ObjectName": "badges_name_idx ON public.badges", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX badges_name_idx ON public.badges USING btree (name) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX badges_name_idx ON public.badges USING btree (name) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -69,9 +69,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "badges_user_id_idx", + "ObjectName": "badges_user_id_idx ON public.badges", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX badges_user_id_idx ON public.badges USING btree (userid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX badges_user_id_idx ON public.badges USING btree (userid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -81,9 +81,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "cmnts_creation_date_idx", + "ObjectName": "cmnts_creation_date_idx ON public.comments", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX cmnts_creation_date_idx ON public.comments USING btree (creationdate) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX cmnts_creation_date_idx ON public.comments USING btree (creationdate) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -93,9 +93,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "cmnts_postid_idx", + "ObjectName": "cmnts_postid_idx ON public.comments", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX cmnts_postid_idx ON public.comments USING hash (postid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX cmnts_postid_idx ON public.comments USING hash (postid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -105,9 +105,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "cmnts_score_idx", + "ObjectName": "cmnts_score_idx ON public.comments", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX cmnts_score_idx ON public.comments USING btree (score) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX cmnts_score_idx ON public.comments USING btree (score) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -117,9 +117,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "cmnts_userid_idx", + "ObjectName": "cmnts_userid_idx ON public.comments", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX cmnts_userid_idx ON public.comments USING btree (userid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX cmnts_userid_idx ON public.comments USING btree (userid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -129,9 +129,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "ph_creation_date_idx", + "ObjectName": "ph_creation_date_idx ON public.posthistory", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX ph_creation_date_idx ON public.posthistory USING btree (creationdate) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX ph_creation_date_idx ON public.posthistory USING btree (creationdate) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -141,9 +141,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "ph_post_type_id_idx", + "ObjectName": "ph_post_type_id_idx ON public.posthistory", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX ph_post_type_id_idx ON public.posthistory USING btree (posthistorytypeid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX ph_post_type_id_idx ON public.posthistory USING btree (posthistorytypeid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -153,9 +153,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "ph_postid_idx", + "ObjectName": "ph_postid_idx ON public.posthistory", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX ph_postid_idx ON public.posthistory USING hash (postid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX ph_postid_idx ON public.posthistory USING hash (postid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -165,9 +165,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "ph_revguid_idx", + "ObjectName": "ph_revguid_idx ON public.posthistory", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX ph_revguid_idx ON public.posthistory USING btree (revisionguid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX ph_revguid_idx ON public.posthistory USING btree (revisionguid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -177,9 +177,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "ph_userid_idx", + "ObjectName": "ph_userid_idx ON public.posthistory", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX ph_userid_idx ON public.posthistory USING btree (userid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX ph_userid_idx ON public.posthistory USING btree (userid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -189,9 +189,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "postlinks_post_id_idx", + "ObjectName": "postlinks_post_id_idx ON public.postlinks", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX postlinks_post_id_idx ON public.postlinks USING btree (postid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX postlinks_post_id_idx ON public.postlinks USING btree (postid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -201,9 +201,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "postlinks_related_post_id_idx", + "ObjectName": "postlinks_related_post_id_idx ON public.postlinks", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX postlinks_related_post_id_idx ON public.postlinks USING btree (relatedpostid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX postlinks_related_post_id_idx ON public.postlinks USING btree (relatedpostid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -213,9 +213,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_accepted_answer_id_idx", + "ObjectName": "posts_accepted_answer_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_accepted_answer_id_idx ON public.posts USING btree (acceptedanswerid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_accepted_answer_id_idx ON public.posts USING btree (acceptedanswerid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -225,9 +225,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_answer_count_idx", + "ObjectName": "posts_answer_count_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_answer_count_idx ON public.posts USING btree (answercount) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_answer_count_idx ON public.posts USING btree (answercount) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -237,9 +237,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_comment_count_idx", + "ObjectName": "posts_comment_count_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_comment_count_idx ON public.posts USING btree (commentcount) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_comment_count_idx ON public.posts USING btree (commentcount) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -249,9 +249,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_creation_date_idx", + "ObjectName": "posts_creation_date_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_creation_date_idx ON public.posts USING btree (creationdate) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_creation_date_idx ON public.posts USING btree (creationdate) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -261,9 +261,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_favorite_count_idx", + "ObjectName": "posts_favorite_count_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_favorite_count_idx ON public.posts USING btree (favoritecount) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_favorite_count_idx ON public.posts USING btree (favoritecount) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -273,9 +273,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_id_accepted_answers_id_idx", + "ObjectName": "posts_id_accepted_answers_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_id_accepted_answers_id_idx ON public.posts USING btree (id, acceptedanswerid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_id_accepted_answers_id_idx ON public.posts USING btree (id, acceptedanswerid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -285,9 +285,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_id_parent_id_idx", + "ObjectName": "posts_id_parent_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_id_parent_id_idx ON public.posts USING btree (id, parentid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_id_parent_id_idx ON public.posts USING btree (id, parentid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -297,9 +297,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_id_post_type_id_idx", + "ObjectName": "posts_id_post_type_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_id_post_type_id_idx ON public.posts USING btree (id, posttypeid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_id_post_type_id_idx ON public.posts USING btree (id, posttypeid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -309,9 +309,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_owner_user_id_creation_date_idx", + "ObjectName": "posts_owner_user_id_creation_date_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_owner_user_id_creation_date_idx ON public.posts USING btree (owneruserid, creationdate) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_owner_user_id_creation_date_idx ON public.posts USING btree (owneruserid, creationdate) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -321,9 +321,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_owner_user_id_idx", + "ObjectName": "posts_owner_user_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_owner_user_id_idx ON public.posts USING hash (owneruserid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_owner_user_id_idx ON public.posts USING hash (owneruserid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -333,9 +333,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_parent_id_idx", + "ObjectName": "posts_parent_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_parent_id_idx ON public.posts USING btree (parentid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_parent_id_idx ON public.posts USING btree (parentid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -345,9 +345,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_post_type_id_idx", + "ObjectName": "posts_post_type_id_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_post_type_id_idx ON public.posts USING btree (posttypeid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_post_type_id_idx ON public.posts USING btree (posttypeid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -357,9 +357,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_score_idx", + "ObjectName": "posts_score_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_score_idx ON public.posts USING btree (score) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_score_idx ON public.posts USING btree (score) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -369,9 +369,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posts_viewcount_idx", + "ObjectName": "posts_viewcount_idx ON public.posts", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posts_viewcount_idx ON public.posts USING btree (viewcount) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posts_viewcount_idx ON public.posts USING btree (viewcount) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -381,9 +381,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posttags_postid_idx", + "ObjectName": "posttags_postid_idx ON public.posttags", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posttags_postid_idx ON public.posttags USING hash (postid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posttags_postid_idx ON public.posttags USING hash (postid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -393,9 +393,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "posttags_tagid_idx", + "ObjectName": "posttags_tagid_idx ON public.posttags", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX posttags_tagid_idx ON public.posttags USING btree (tagid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX posttags_tagid_idx ON public.posttags USING btree (tagid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -405,9 +405,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "tags_count_idx", + "ObjectName": "tags_count_idx ON public.tags", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX tags_count_idx ON public.tags USING btree (count) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX tags_count_idx ON public.tags USING btree (count) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -417,9 +417,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "tags_name_idx", + "ObjectName": "tags_name_idx ON public.tags", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX tags_name_idx ON public.tags USING hash (tagname) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX tags_name_idx ON public.tags USING hash (tagname) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -429,9 +429,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "user_acc_id_idx", + "ObjectName": "user_acc_id_idx ON public.users", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX user_acc_id_idx ON public.users USING hash (accountid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX user_acc_id_idx ON public.users USING hash (accountid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -441,9 +441,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "user_created_at_idx", + "ObjectName": "user_created_at_idx ON public.users", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX user_created_at_idx ON public.users USING btree (creationdate) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX user_created_at_idx ON public.users USING btree (creationdate) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -453,9 +453,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "user_display_idx", + "ObjectName": "user_display_idx ON public.users", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX user_display_idx ON public.users USING hash (displayname) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX user_display_idx ON public.users USING hash (displayname) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -465,9 +465,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "user_down_votes_idx", + "ObjectName": "user_down_votes_idx ON public.users", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX user_down_votes_idx ON public.users USING btree (downvotes) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX user_down_votes_idx ON public.users USING btree (downvotes) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -477,9 +477,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "user_up_votes_idx", + "ObjectName": "user_up_votes_idx ON public.users", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX user_up_votes_idx ON public.users USING btree (upvotes) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX user_up_votes_idx ON public.users USING btree (upvotes) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -489,9 +489,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "usertagqa_all_qa_posts_idx", + "ObjectName": "usertagqa_all_qa_posts_idx ON public.usertagqa", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX usertagqa_all_qa_posts_idx ON public.usertagqa USING btree (((questions + answers))) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX usertagqa_all_qa_posts_idx ON public.usertagqa USING btree (((questions + answers))) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -501,9 +501,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "usertagqa_answers_idx", + "ObjectName": "usertagqa_answers_idx ON public.usertagqa", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX usertagqa_answers_idx ON public.usertagqa USING btree (answers) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX usertagqa_answers_idx ON public.usertagqa USING btree (answers) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -513,9 +513,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "usertagqa_questions_answers_idx", + "ObjectName": "usertagqa_questions_answers_idx ON public.usertagqa", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX usertagqa_questions_answers_idx ON public.usertagqa USING btree (questions, answers) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX usertagqa_questions_answers_idx ON public.usertagqa USING btree (questions, answers) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -525,9 +525,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "usertagqa_questions_idx", + "ObjectName": "usertagqa_questions_idx ON public.usertagqa", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX usertagqa_questions_idx ON public.usertagqa USING btree (questions) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX usertagqa_questions_idx ON public.usertagqa USING btree (questions) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -537,9 +537,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "votes_creation_date_idx", + "ObjectName": "votes_creation_date_idx ON public.votes", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX votes_creation_date_idx ON public.votes USING btree (creationdate) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX votes_creation_date_idx ON public.votes USING btree (creationdate) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -549,9 +549,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "votes_post_id_idx", + "ObjectName": "votes_post_id_idx ON public.votes", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX votes_post_id_idx ON public.votes USING hash (postid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX votes_post_id_idx ON public.votes USING hash (postid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", @@ -561,9 +561,9 @@ { "IssueType": "unsupported_features", "ObjectType": "INDEX", - "ObjectName": "votes_type_idx", + "ObjectName": "votes_type_idx ON public.votes", "Reason": "Storage parameters are not supported yet.", - "SqlStatement": "CREATE INDEX votes_type_idx ON public.votes USING btree (votetypeid) WITH (fillfactor='100'); ", + "SqlStatement": "CREATE INDEX votes_type_idx ON public.votes USING btree (votetypeid) WITH (fillfactor='100');", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/stackexchange/export-dir/schema/tables/INDEXES_table.sql", "Suggestion": "Remove the storage parameters from the DDL", "GH": "https://github.com/yugabyte/yugabyte-db/issues/23467", diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 57483aeb76..a60f0e8ae9 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -38,6 +38,7 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/queryissue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" @@ -129,32 +130,10 @@ var ( schemaAnalysisReport utils.SchemaReport partitionTablesMap = make(map[string]bool) // key is partitioned table, value is sqlInfo (sqlstmt, fpath) where the ADD PRIMARY KEY statement resides - primaryConsInAlter = make(map[string]*sqlInfo) summaryMap = make(map[string]*summaryInfo) parserIssueDetector = queryissue.NewParserIssueDetector() multiRegex = regexp.MustCompile(`([a-zA-Z0-9_\.]+[,|;])`) dollarQuoteRegex = regexp.MustCompile(`(\$.*\$)`) - /* - this will contain the information in this format: - public.table1 -> { - column1: citext | jsonb | inet | tsquery | tsvector | array - ... - } - schema2.table2 -> { - column3: citext | jsonb | inet | tsquery | tsvector | array - ... - } - Here only those columns on tables are stored which have unsupported type for Index in YB - */ - columnsWithUnsupportedIndexDatatypes = make(map[string]map[string]string) - /* - list of composite types with fully qualified typename in the exported schema - */ - compositeTypes = make([]string, 0) - /* - list of enum types with fully qualified typename in the exported schema - */ - enumTypes = make([]string, 0) //TODO: optional but replace every possible space or new line char with [\s\n]+ in all regexs viewWithCheckRegex = re("VIEW", capture(ident), anything, "WITH", opt(commonClause), "CHECK", "OPTION") rangeRegex = re("PRECEDING", "and", anything, ":float") @@ -172,7 +151,6 @@ var ( idxConcRegex = re("REINDEX", anything, capture(ident)) likeAllRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "LIKE", anything, "INCLUDING ALL") likeRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, `\(LIKE`) - inheritRegex = re("CREATE", opt(capture(unqualifiedIdent)), "TABLE", ifNotExists, capture(ident), anything, "INHERITS", "[ |(]") withOidsRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "WITH", anything, "OIDS") anydataRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "AnyData", anything) anydatasetRegex = re("CREATE", "TABLE", ifNotExists, capture(ident), anything, "AnyDataSet", anything) @@ -345,31 +323,9 @@ func addSummaryDetailsForIndexes() { } } -func checkForeignTable(sqlInfoArr []sqlInfo, fpath string) { - for _, sqlStmtInfo := range sqlInfoArr { - parseTree, err := pg_query.Parse(sqlStmtInfo.stmt) - if err != nil { - utils.ErrExit("failed to parse the stmt %v: %v", sqlStmtInfo.stmt, err) - } - createForeignTableNode, isForeignTable := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateForeignTableStmt) - if isForeignTable { - baseStmt := createForeignTableNode.CreateForeignTableStmt.BaseStmt - relation := baseStmt.Relation - schemaName := relation.Schemaname - tableName := relation.Relname - serverName := createForeignTableNode.CreateForeignTableStmt.Servername - summaryMap["FOREIGN TABLE"].invalidCount[sqlStmtInfo.objName] = true - objName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - reportCase(fpath, FOREIGN_TABLE_ISSUE_REASON, "https://github.com/yugabyte/yb-voyager/issues/1627", - fmt.Sprintf("SERVER '%s', and USER MAPPING should be created manually on the target to create and use the foreign table", serverName), "FOREIGN TABLE", objName, sqlStmtInfo.stmt, MIGRATION_CAVEATS, FOREIGN_TABLE_DOC_LINK) - reportUnsupportedDatatypes(relation, baseStmt.TableElts, sqlStmtInfo, fpath, "FOREIGN TABLE") - } - } -} - func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { for _, sqlStmtInfo := range sqlInfoArr { - parseTree, err := pg_query.Parse(sqlStmtInfo.stmt) + _, err := queryparser.Parse(sqlStmtInfo.stmt) if err != nil { //if the Stmt is not already report by any of the regexes if !summaryMap[objType].invalidCount[sqlStmtInfo.objName] { reason := fmt.Sprintf("%s - '%s'", UNSUPPORTED_PG_SYNTAX, err.Error()) @@ -378,1090 +334,21 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { } continue } - createTableNode, isCreateTable := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateStmt) - alterTableNode, isAlterTable := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_AlterTableStmt) - createIndexNode, isCreateIndex := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_IndexStmt) - createPolicyNode, isCreatePolicy := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreatePolicyStmt) - createCompositeTypeNode, isCreateCompositeType := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CompositeTypeStmt) - createEnumTypeNode, isCreateEnumType := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateEnumStmt) - createTriggerNode, isCreateTrigger := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTrigStmt) - - if objType == TABLE && isCreateTable { - reportPartitionsRelatedIssues(createTableNode, sqlStmtInfo, fpath) - reportGeneratedStoredColumnTables(createTableNode, sqlStmtInfo, fpath) - reportExclusionConstraintCreateTable(createTableNode, sqlStmtInfo, fpath) - reportDeferrableConstraintCreateTable(createTableNode, sqlStmtInfo, fpath) - reportUnsupportedDatatypes(createTableNode.CreateStmt.Relation, createTableNode.CreateStmt.TableElts, sqlStmtInfo, fpath, objType) - parseColumnsWithUnsupportedIndexDatatypes(createTableNode) - reportUnsupportedConstraintsOnComplexDatatypesInCreate(createTableNode, sqlStmtInfo, fpath) - reportUnloggedTable(createTableNode, sqlStmtInfo, fpath) - } - if isAlterTable { - reportUnsupportedConstraintsOnComplexDatatypesInAlter(alterTableNode, sqlStmtInfo, fpath) - reportAlterAddPKOnPartition(alterTableNode, sqlStmtInfo, fpath) - reportAlterTableVariants(alterTableNode, sqlStmtInfo, fpath, objType) - reportExclusionConstraintAlterTable(alterTableNode, sqlStmtInfo, fpath) - reportDeferrableConstraintAlterTable(alterTableNode, sqlStmtInfo, fpath) - } - if isCreateIndex { - reportIndexMethods(createIndexNode, sqlStmtInfo, fpath) - reportCreateIndexStorageParameter(createIndexNode, sqlStmtInfo, fpath) - reportUnsupportedIndexesOnComplexDatatypes(createIndexNode, sqlStmtInfo, fpath) - checkGinVariations(createIndexNode, sqlStmtInfo, fpath) - } - - if isCreatePolicy { - reportPolicyRequireRolesOrGrants(createPolicyNode, sqlStmtInfo, fpath) - } - - if isCreateTrigger { - reportUnsupportedTriggers(createTriggerNode, sqlStmtInfo, fpath) - } - - if isCreateCompositeType { - //Adding the composite types (UDTs) in the list - /* - e.g. CREATE TYPE non_public."Address_type" AS ( - street VARCHAR(100), - city VARCHAR(50), - state VARCHAR(50), - zip_code VARCHAR(10) - ); - stmt:{composite_type_stmt:{typevar:{schemaname:"non_public" relname:"Address_type" relpersistence:"p" location:14} coldeflist:{column_def:{colname:"street" - type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"varchar"}} typmods:{a_const:{ival:{ival:100} location:65}} typemod:-1 location:57} ... - - Here the type name is required which is available in typevar->relname typevar->schemaname for qualified name - */ - typeName := createCompositeTypeNode.CompositeTypeStmt.Typevar.GetRelname() - typeSchemaName := createCompositeTypeNode.CompositeTypeStmt.Typevar.GetSchemaname() - fullTypeName := lo.Ternary(typeSchemaName != "", typeSchemaName+"."+typeName, typeName) - compositeTypes = append(compositeTypes, fullTypeName) - } - if isCreateEnumType { - //Adding the composite types (UDTs) in the list - /* - e.g. CREATE TYPE decline_reason AS ENUM ( - 'duplicate_payment_method', - 'server_failure' - ); - stmt:{create_enum_stmt:{type_name:{string:{sval:"decline_reason"}} vals:{string:{sval:"duplicate_payment_method"}} vals:{string:{sval:"server_failure"}}}} - stmt_len:101} - - Here the type name is required which is available in typevar->relname typevar->schemaname for qualified name - */ - typeNames := createEnumTypeNode.CreateEnumStmt.GetTypeName() - typeName, typeSchemaName := getTypeNameAndSchema(typeNames) - fullTypeName := lo.Ternary(typeSchemaName != "", typeSchemaName+"."+typeName, typeName) - enumTypes = append(enumTypes, fullTypeName) - } - } -} - -func reportUnsupportedTriggers(createTriggerNode *pg_query.Node_CreateTrigStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := createTriggerNode.CreateTrigStmt.Relation.Schemaname - tableName := createTriggerNode.CreateTrigStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - trigName := createTriggerNode.CreateTrigStmt.Trigname - displayObjectName := fmt.Sprintf("%s ON %s", trigName, fullyQualifiedName) - - /* - e.g.CREATE CONSTRAINT TRIGGER some_trig - AFTER DELETE ON xyz_schema.abc - DEFERRABLE INITIALLY DEFERRED - FOR EACH ROW EXECUTE PROCEDURE xyz_schema.some_trig(); - create_trig_stmt:{isconstraint:true trigname:"some_trig" relation:{schemaname:"xyz_schema" relname:"abc" inh:true relpersistence:"p" - location:56} funcname:{string:{sval:"xyz_schema"}} funcname:{string:{sval:"some_trig"}} row:true events:8 deferrable:true initdeferred:true}} - stmt_len:160} - */ - if createTriggerNode.CreateTrigStmt.Isconstraint { - reportCase(fpath, CONSTRAINT_TRIGGER_ISSUE_REASON, - "https://github.com/YugaByte/yugabyte-db/issues/1709", "", "TRIGGER", displayObjectName, - sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CONSTRAINT_TRIGGER_DOC_LINK) - } - - /* - e.g. CREATE TRIGGER projects_loose_fk_trigger - AFTER DELETE ON public.projects - REFERENCING OLD TABLE AS old_table - FOR EACH STATEMENT EXECUTE FUNCTION xyz_schema.some_trig(); - stmt:{create_trig_stmt:{trigname:"projects_loose_fk_trigger" relation:{schemaname:"public" relname:"projects" inh:true - relpersistence:"p" location:58} funcname:{string:{sval:"xyz_schema"}} funcname:{string:{sval:"some_trig"}} events:8 - transition_rels:{trigger_transition:{name:"old_table" is_table:true}}}} stmt_len:167} - */ - if createTriggerNode.CreateTrigStmt.GetTransitionRels() != nil { - summaryMap["TRIGGER"].invalidCount[displayObjectName] = true - reportCase(fpath, REFERENCING_CLAUSE_FOR_TRIGGERS, - "https://github.com/YugaByte/yugabyte-db/issues/1668", "", "TRIGGER", displayObjectName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, REFERENCING_CLAUSE_TRIGGER_DOC_LINK) - } - - /* - e.g.CREATE TRIGGER after_insert_or_delete_trigger - BEFORE INSERT OR DELETE ON main_table - FOR EACH ROW - EXECUTE FUNCTION handle_insert_or_delete(); - stmt:{create_trig_stmt:{trigname:"after_insert_or_delete_trigger" relation:{relname:"main_table" inh:true relpersistence:"p" - location:111} funcname:{string:{sval:"handle_insert_or_delete"}} row:true timing:2 events:12}} stmt_len:177} - - here, - timing - bits of BEFORE/AFTER/INSTEAD - events - bits of "OR" INSERT/UPDATE/DELETE/TRUNCATE - row - FOR EACH ROW (true), FOR EACH STATEMENT (false) - refer - https://github.com/pganalyze/pg_query_go/blob/c3a818d346a927c18469460bb18acb397f4f4301/parser/include/postgres/catalog/pg_trigger_d.h#L49 - TRIGGER_TYPE_BEFORE (1 << 1) - TRIGGER_TYPE_INSERT (1 << 2) - TRIGGER_TYPE_DELETE (1 << 3) - TRIGGER_TYPE_UPDATE (1 << 4) - TRIGGER_TYPE_TRUNCATE (1 << 5) - TRIGGER_TYPE_INSTEAD (1 << 6) - */ - - timing := createTriggerNode.CreateTrigStmt.Timing - isSecondBitSet := timing&(1<<1) != 0 - if isSecondBitSet && createTriggerNode.CreateTrigStmt.Row { - // BEFORE clause will have the bits in timing as 1<<1 - // BEFORE and FOR EACH ROW on partitioned table is not supported in PG<=12 - if partitionTablesMap[fullyQualifiedName] { - summaryMap["TRIGGER"].invalidCount[displayObjectName] = true - reportCase(fpath, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, - "https://github.com/yugabyte/yugabyte-db/issues/24830", "Create the triggers on individual partitions.", "TRIGGER", displayObjectName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, BEFORE_ROW_TRIGGER_PARTITIONED_TABLE_DOC_LINK) - } - } - -} - -func reportAlterAddPKOnPartition(alterTableNode *pg_query.Node_AlterTableStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := alterTableNode.AlterTableStmt.Relation.Schemaname - tableName := alterTableNode.AlterTableStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - - alterCmd := alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd() - /* - e.g. - ALTER TABLE example2 - ADD CONSTRAINT example2_pkey PRIMARY KEY (id); - tmts:{stmt:{alter_table_stmt:{relation:{relname:"example2" inh:true relpersistence:"p" location:693} - cmds:{alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_PRIMARY conname:"example2_pkey" - location:710 keys:{string:{sval:"id"}}}} behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} stmt_location:679 stmt_len:72} - - */ - - constraint := alterCmd.GetDef().GetConstraint() - - if constraint != nil && constraint.Contype == pg_query.ConstrType_CONSTR_PRIMARY { - if partitionTablesMap[fullyQualifiedName] { - reportCase(fpath, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, - "https://github.com/yugabyte/yugabyte-db/issues/10074", "", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, MIGRATION_CAVEATS, ADDING_PK_TO_PARTITIONED_TABLE_DOC_LINK) - } else { - primaryConsInAlter[fullyQualifiedName] = &sqlStmtInfo - } - } -} - -/* -This functions reports multiple issues - -1. Adding PK to Partitioned Table (in cases where ALTER is before create) -2. Expression partitions are not allowed if PK/UNIQUE columns are there is table -3. List partition strategy is not allowed with multi-column partitions. -4. Partition columns should all be included in Primary key set if any on table. -*/ -func reportPartitionsRelatedIssues(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - columns := createTableNode.CreateStmt.TableElts - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - - /* - e.g. In case if PRIMARY KEY is included in column definition - CREATE TABLE example2 ( - id numeric NOT NULL PRIMARY KEY, - country_code varchar(3), - record_type varchar(5) - ) PARTITION BY RANGE (country_code, record_type) ; - stmts:{stmt:{create_stmt:{relation:{relname:"example2" inh:true relpersistence:"p" location:193} table_elts:{column_def:{colname:"id" - type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"numeric"}} typemod:-1 location:208} is_local:true - constraints:{constraint:{contype:CONSTR_NOTNULL location:216}} constraints:{constraint:{contype:CONSTR_PRIMARY location:225}} - location:205}} ... partspec:{strategy:PARTITION_STRATEGY_RANGE - part_params:{partition_elem:{name:"country_code" location:310}} part_params:{partition_elem:{name:"record_type" location:324}} - location:290} oncommit:ONCOMMIT_NOOP}} stmt_location:178 stmt_len:159} - - In case if PRIMARY KEY in column list CREATE TABLE example1 (..., PRIMARY KEY(id,country_code) ) PARTITION BY RANGE (country_code, record_type); - stmts:{stmt:{create_stmt:{relation:{relname:"example1" inh:true relpersistence:"p" location:15} table_elts:{column_def:{colname:"id" - type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"numeric"}} ... table_elts:{constraint:{contype:CONSTR_PRIMARY - location:98 keys:{string:{sval:"id"}} keys:{string:{sval:"country_code"}}}} partspec:{strategy:PARTITION_STRATEGY_RANGE - part_params:{partition_elem:{name:"country_code" location:150}} part_params:{partition_elem:{name:"record_type" ... - */ - if createTableNode.CreateStmt.GetPartspec() == nil { - //If not partition table then no need to proceed - return - } - - if primaryConsInAlter[fullyQualifiedName] != nil { - //reporting the ALTER TABLE ADD PK on partition table here in case the order is different if ALTER is before the CREATE - alterTableSqlInfo := primaryConsInAlter[fullyQualifiedName] - reportCase(alterTableSqlInfo.fileName, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, - "https://github.com/yugabyte/yugabyte-db/issues/10074", "", "TABLE", fullyQualifiedName, alterTableSqlInfo.formattedStmt, MIGRATION_CAVEATS, ADDING_PK_TO_PARTITIONED_TABLE_DOC_LINK) - } - - partitionTablesMap[fullyQualifiedName] = true // marking the partition tables in the map - - var primaryKeyColumns, partitionColumns, uniqueKeyColumns []string - - for _, column := range columns { - if column.GetColumnDef() != nil { //In case PRIMARY KEY constraint is added with column definition - constraints := column.GetColumnDef().Constraints - for _, constraint := range constraints { - if constraint.GetConstraint().Contype == pg_query.ConstrType_CONSTR_PRIMARY { - primaryKeyColumns = []string{column.GetColumnDef().Colname} - } - if constraint.GetConstraint().Contype == pg_query.ConstrType_CONSTR_UNIQUE { - uniqueKeyColumns = append(uniqueKeyColumns, column.GetColumnDef().Colname) - } - } - } else if column.GetConstraint() != nil { - //In case CREATE DDL has PRIMARY KEY(column_name) - it will be included in columns but won't have columnDef as its a constraint - for _, key := range column.GetConstraint().GetKeys() { - if column.GetConstraint().Contype == pg_query.ConstrType_CONSTR_PRIMARY { - primaryKeyColumns = append(primaryKeyColumns, key.GetString_().Sval) - } else if column.GetConstraint().Contype == pg_query.ConstrType_CONSTR_UNIQUE { - uniqueKeyColumns = append(uniqueKeyColumns, key.GetString_().Sval) - } - } - } - } - - partitionElements := createTableNode.CreateStmt.GetPartspec().GetPartParams() - - for _, partElem := range partitionElements { - if partElem.GetPartitionElem().GetExpr() != nil { - //Expression partitions - if len(primaryKeyColumns) > 0 || len(uniqueKeyColumns) > 0 { - summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true - reportCase(fpath, "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", - "https://github.com/yugabyte/yb-voyager/issues/698", "Remove the Constriant from the table definition", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, EXPRESSION_PARTIITON_DOC_LINK) - } - } else { - partitionColumns = append(partitionColumns, partElem.GetPartitionElem().GetName()) - } - } - - if len(partitionColumns) > 1 && createTableNode.CreateStmt.GetPartspec().GetStrategy() == pg_query.PartitionStrategy_PARTITION_STRATEGY_LIST { - summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true - reportCase(fpath, `cannot use "list" partition strategy with more than one column`, - "https://github.com/yugabyte/yb-voyager/issues/699", "Make it a single column partition by list or choose other supported Partitioning methods", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, LIST_PARTIION_MULTI_COLUMN_DOC_LINK) - } - - if len(primaryKeyColumns) == 0 { // no need to report in case of non-PK tables - return - } - - partitionColumnsNotInPK, _ := lo.Difference(partitionColumns, primaryKeyColumns) - if len(partitionColumnsNotInPK) > 0 { - summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true - reportCase(fpath, fmt.Sprintf("%s - (%s)", INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, strings.Join(partitionColumnsNotInPK, ", ")), - "https://github.com/yugabyte/yb-voyager/issues/578", "Add all Partition columns to Primary Key", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, PARTITION_KEY_NOT_PK_DOC_LINK) - } - -} - -// Reference for some of the types https://docs.yugabyte.com/stable/api/ysql/datatypes/ (datatypes with type 1) -var UnsupportedIndexDatatypes = []string{ - "citext", - "tsvector", - "tsquery", - "jsonb", - "inet", - "json", - "macaddr", - "macaddr8", - "cidr", - "bit", // for BIT (n) - "varbit", // for BIT varying (n) - "daterange", - "tsrange", - "tstzrange", - "numrange", - "int4range", - "int8range", - "interval", // same for INTERVAL YEAR TO MONTH and INTERVAL DAY TO SECOND - //Below ones are not supported on PG as well with atleast btree access method. Better to have in our list though - //Need to understand if there is other method or way available in PG to have these index key [TODO] - "circle", - "box", - "line", - "lseg", - "point", - "pg_lsn", - "path", - "polygon", - "txid_snapshot", - // array as well but no need to add it in the list as fetching this type is a different way TODO: handle better with specific types -} - -func reportUnsupportedConstraintsOnComplexDatatypesInAlter(alterTableNode *pg_query.Node_AlterTableStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := alterTableNode.AlterTableStmt.Relation.Schemaname - tableName := alterTableNode.AlterTableStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - alterCmd := alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd() - unsupportedColumnsForTable, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] - if !ok { - return - } - if alterCmd.GetSubtype() != pg_query.AlterTableType_AT_AddConstraint { - return - } - if !slices.Contains([]pg_query.ConstrType{pg_query.ConstrType_CONSTR_PRIMARY, pg_query.ConstrType_CONSTR_UNIQUE}, - alterCmd.GetDef().GetConstraint().GetContype()) { - return - } - columns := alterCmd.GetDef().GetConstraint().GetKeys() - for _, col := range columns { - colName := col.GetString_().Sval - typeName, ok := unsupportedColumnsForTable[colName] - if ok { - displayName := fmt.Sprintf("%s, constraint: %s", fullyQualifiedName, alterCmd.GetDef().GetConstraint().GetConname()) - reportCase(fpath, fmt.Sprintf(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/25003", - "Refer to the docs link for the workaround", "TABLE", displayName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, PK_UK_CONSTRAINT_ON_UNSUPPORTED_TYPE) - return - } - } -} - -func reportUnsupportedConstraintsOnComplexDatatypesInCreate(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - columns := createTableNode.CreateStmt.TableElts - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - unsupportedColumnsForTable, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] - if !ok { - return - } - reportConstraintIfrequired := func(con *pg_query.Constraint, colNames []string, typeName string) { - conType := con.GetContype() - if !slices.Contains([]pg_query.ConstrType{pg_query.ConstrType_CONSTR_PRIMARY, pg_query.ConstrType_CONSTR_UNIQUE}, conType) { - return - } - generatedConName := generateConstraintName(conType, fullyQualifiedName, colNames) - specifiedConstraintName := con.GetConname() - conName := lo.Ternary(specifiedConstraintName == "", generatedConName, specifiedConstraintName) - //report the PK / Unique constraint in CREATE TABLE on this column - summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true - reportCase(fpath, fmt.Sprintf(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/25003", - "Refer to the docs link for the workaround", "TABLE", fmt.Sprintf("%s, constraint: %s", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, PK_UK_CONSTRAINT_ON_UNSUPPORTED_TYPE) - - } - for _, column := range columns { - if column.GetColumnDef() != nil { - /* - e.g. create table unique_def_test(id int, d daterange UNIQUE, c1 int); - create_stmt:{relation:{relname:"unique_def_test" inh:true relpersistence:"p" location:15}... - table_elts:{column_def:{colname:"d" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} - typemod:-1 location:34} is_local:true constraints:{constraint:{contype:CONSTR_UNIQUE location:38}} .... - - here checking the case where this clause is in column definition so iterating over each column_def and in that - constraint type is UNIQUE/ PK reporting that - supported. - */ - colName := column.GetColumnDef().GetColname() - typeName, ok := unsupportedColumnsForTable[colName] - if !ok { - continue - } - constraints := column.GetColumnDef().GetConstraints() - for _, c := range constraints { - reportConstraintIfrequired(c.GetConstraint(), []string{colName}, typeName) - } - } else if column.GetConstraint() != nil { - /* - e.g. create table uniquen_def_test1(id int, c1 citext, CONSTRAINT pk PRIMARY KEY(id, c)); - {create_stmt:{relation:{relname:"unique_def_test1" inh:true relpersistence:"p" location:80} table_elts:{column_def:{colname:"id" - type_name:{.... names:{string:{sval:"int4"}} typemod:-1 location:108} is_local:true location:105}} - table_elts:{constraint:{contype:CONSTR_UNIQUE deferrable:true initdeferred:true location:113 keys:{string:{sval:"id"}}}} .. - - here checking the case where this UK/ PK is at the end of column definition as a separate constraint - */ - keys := column.GetConstraint().GetKeys() - columns := []string{} - for _, k := range keys { - colName := k.GetString_().Sval - columns = append(columns, colName) - } - for _, c := range columns { - typeName, ok := unsupportedColumnsForTable[c] - if !ok { - continue - } - reportConstraintIfrequired(column.GetConstraint(), columns, typeName) - } - } - } -} - -func parseColumnsWithUnsupportedIndexDatatypes(createTableNode *pg_query.Node_CreateStmt) { - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - columns := createTableNode.CreateStmt.TableElts - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - for _, column := range columns { - /* - e.g. 1. CREATE TABLE public.citext_type ( - id integer, - lists_of_data text[], - data public.citext - ); - stmt:{create_stmt:{relation:{schemaname:"public" relname:"citext_type" inh:true relpersistence:"p" location:258} table_elts:{column_def:{colname:"id" - type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:287} is_local:true location:284}} table_elts: - {column_def:{colname:"lists_of_data" type_name:{names:{string:{sval:"text"}} typemod:-1 array_bounds:{integer:{ival:-1}} location:315} is_local:true - location:301}} table_elts:{column_def:{colname:"data" type_name:{names:{string:{sval:"public"}} names:{string:{sval:"citext"}} typemod:-1 location:333} - is_local:true location:328}} oncommit:ONCOMMIT_NOOP}} stmt_location:244 stmt_len:108 - - 2. CREATE TABLE public.ts_query_table ( - id int generated by default as identity, - query tsquery - ); - stmt:{create_stmt:{relation:{schemaname:"public" relname:"ts_query_table" inh:true relpersistence:"p" location:211} table_elts:{column_def:{colname:"id" - type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:242} is_local:true constraints:{constraint:{contype:CONSTR_IDENTITY - location:246 generated_when:"d"}} location:239}} table_elts:{column_def:{colname:"query" type_name:{names:{string:{sval:"tsquery"}} - typemod:-1 location:290} is_local:true location:284}} oncommit:ONCOMMIT_NOOP}} stmt_location:196 stmt_len:110 - - 3. create table combined_tbl ( - id int, c cidr, ci circle, b box, j json, - l line, ls lseg, maddr macaddr, maddr8 macaddr8, p point, - lsn pg_lsn, p1 path, p2 polygon, id1 txid_snapshot, - bitt bit (13), bittv bit varying(15), address non_public."Address_type" - ); - stmt:{create_stmt:{relation:{relname:"combined_tbl" ... colname:"id" type_name:...names:{string:{sval:"int4"}}... column_def:{colname:"c" type_name:{names:{string:{sval:"cidr"}} - ... column_def:{colname:"ci" type_name:{names:{string:{sval:"circle"}} ... column_def:{colname:"b"type_name:{names:{string:{sval:"box"}} ... column_def:{colname:"j" type_name:{names:{string:{sval:"json"}} - ... column_def:{colname:"l" type_name:{names:{string:{sval:"line"}} ...column_def:{colname:"ls" type_name:{names:{string:{sval:"lseg"}} ...column_def:{colname:"maddr" type_name:{names:{string:{sval:"macaddr"}} - ...column_def:{colname:"maddr8" type_name:{names:{string:{sval:"macaddr8"}}...column_def:{colname:"p" type_name:{names:{string:{sval:"point"}} ...column_def:{colname:"lsn" type_name:{names:{string:{sval:"pg_lsn"}} - ...column_def:{colname:"p1" type_name:{names:{string:{sval:"path"}} .... column_def:{colname:"p2" type_name:{names:{string:{sval:"polygon"}} .... column_def:{colname:"id1" type_name:{names:{string:{sval:"txid_snapshot"}} - ... column_def:{colname:"bitt" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"bit"}} typmods:{a_const:{ival:{ival:13} location:241}} typemod:-1 location:236} is_local:true location:231}} - table_elts:{column_def:{colname:"bittv" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"varbit"}} typmods:{a_const:{ival:{ival:15} location:264}} typemod:-1 location:252} ... column_def:{colname:"address" - type_name:{names:{string:{sval:"non_public"}} names:{string:{sval:"Address_type"}} is_local:true location:246}} oncommit:ONCOMMIT_NOOP}} stmt_location:51 stmt_len:217 - - - */ - if column.GetColumnDef() != nil { - typeNames := column.GetColumnDef().GetTypeName().GetNames() - typeName, typeSchemaName := getTypeNameAndSchema(typeNames) - fullTypeName := lo.Ternary(typeSchemaName != "", typeSchemaName+"."+typeName, typeName) - colName := column.GetColumnDef().GetColname() - isArrayType := len(column.GetColumnDef().GetTypeName().GetArrayBounds()) > 0 - isUnsupportedType := slices.Contains(UnsupportedIndexDatatypes, typeName) - isUDTType := slices.Contains(compositeTypes, fullTypeName) - switch true { - case isArrayType: - //For Array types and storing the type as "array" as of now we can enhance the to have specific type e.g. INT4ARRAY - _, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] - if !ok { - columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] = make(map[string]string) - } - columnsWithUnsupportedIndexDatatypes[fullyQualifiedName][colName] = "array" - case isUnsupportedType || isUDTType: - _, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] - if !ok { - columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] = make(map[string]string) - } - columnsWithUnsupportedIndexDatatypes[fullyQualifiedName][colName] = typeName - if slices.Contains(compositeTypes, fullTypeName) { //For UDTs - columnsWithUnsupportedIndexDatatypes[fullyQualifiedName][colName] = "user_defined_type" - } - } - } - } -} - -func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_IndexStmt, sqlStmtInfo sqlInfo, fpath string) { - indexName := createIndexNode.IndexStmt.GetIdxname() - relName := createIndexNode.IndexStmt.GetRelation() - schemaName := relName.GetSchemaname() - tableName := relName.GetRelname() - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - displayObjName := fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName) - /* - e.g. - 1. CREATE INDEX tsvector_idx ON public.documents (title_tsvector, id); - stmt:{index_stmt:{idxname:"tsvector_idx" relation:{schemaname:"public" relname:"documents" inh:true relpersistence:"p" location:510} access_method:"btree" - index_params:{index_elem:{name:"title_tsvector" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} index_params:{index_elem:{name:"id" - ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}}}} stmt_location:479 stmt_len:69 - - 2. CREATE INDEX idx_json ON public.test_json ((data::jsonb)); - stmt:{index_stmt:{idxname:"idx_json" relation:{schemaname:"public" relname:"test_json" inh:true relpersistence:"p" location:703} access_method:"btree" - index_params:{index_elem:{expr:{type_cast:{arg:{column_ref:{fields:{string:{sval:"data"}} location:722}} type_name:{names:{string:{sval:"jsonb"}} typemod:-1 - location:728} location:726}} ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}}}} stmt_location:676 stmt_len:59 - */ - if createIndexNode.IndexStmt.AccessMethod != "btree" { - return // Right now not reporting any other access method issues with such types. - } - _, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName] - if !ok { - return - } - for _, param := range createIndexNode.IndexStmt.GetIndexParams() { - /* - cases to cover - 1. normal index on column with these types - 2. expression index with casting of unsupported column to supported types [No handling as such just to test as colName will not be there] - 3. expression index with casting to unsupported types - 4. normal index on column with UDTs - 5. these type of indexes on different access method like gin etc.. [TODO to explore more, for now not reporting the indexes on anyother access method than btree] - */ - colName := param.GetIndexElem().GetName() - typeName, ok := columnsWithUnsupportedIndexDatatypes[fullyQualifiedName][colName] - if ok { - summaryMap["INDEX"].invalidCount[displayObjName] = true - reportCase(fpath, fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, typeName), "https://github.com/yugabyte/yugabyte-db/issues/25003", - "Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, INDEX_ON_UNSUPPORTED_TYPE) - return - } - //For the expression index case to report in case casting to unsupported types #3 - typeNames := param.GetIndexElem().GetExpr().GetTypeCast().GetTypeName().GetNames() - castTypeName, castTypeSchemaName := getTypeNameAndSchema(typeNames) - fullCastTypeName := lo.Ternary(castTypeSchemaName != "", castTypeSchemaName+"."+castTypeName, castTypeName) - if len(param.GetIndexElem().GetExpr().GetTypeCast().GetTypeName().GetArrayBounds()) > 0 { - //In case casting is happening for an array type - summaryMap["INDEX"].invalidCount[displayObjName] = true - reportCase(fpath, fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "array"), "https://github.com/yugabyte/yugabyte-db/issues/25003", - "Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, INDEX_ON_UNSUPPORTED_TYPE) - return - } else if slices.Contains(UnsupportedIndexDatatypes, castTypeName) || slices.Contains(compositeTypes, fullCastTypeName) { - summaryMap["INDEX"].invalidCount[displayObjName] = true - reason := fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, castTypeName) - if slices.Contains(compositeTypes, fullCastTypeName) { - reason = fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "user_defined_type") - } - reportCase(fpath, reason, "https://github.com/yugabyte/yugabyte-db/issues/25003", - "Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, INDEX_ON_UNSUPPORTED_TYPE) - return - } - } -} - -var unsupportedIndexMethods = []string{ - "gist", - "brin", - "spgist", -} - -func reportIndexMethods(createIndexNode *pg_query.Node_IndexStmt, sqlStmtInfo sqlInfo, fpath string) { - indexMethod := createIndexNode.IndexStmt.AccessMethod - - if !slices.Contains(unsupportedIndexMethods, indexMethod) { - return - } - - indexName := createIndexNode.IndexStmt.GetIdxname() - relName := createIndexNode.IndexStmt.GetRelation() - schemaName := relName.GetSchemaname() - tableName := relName.GetRelname() - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - displayObjName := fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName) - - summaryMap["INDEX"].invalidCount[displayObjName] = true - - reportCase(fpath, fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, strings.ToUpper(indexMethod)), - "https://github.com/YugaByte/yugabyte-db/issues/1337", "", "INDEX", displayObjName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, UNSUPPORTED_INDEX_METHODS_DOC_LINK) -} - -func reportUnloggedTable(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - /* - e.g CREATE UNLOGGED TABLE tbl_unlogged (id int, val text); - stmt:{create_stmt:{relation:{schemaname:"public" relname:"tbl_unlogged" inh:true relpersistence:"u" location:19} - table_elts:{column_def:{colname:"id" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} - typemod:-1 location:54} is_local:true location:51}} table_elts:{column_def:{colname:"val" type_name:{names:{string:{sval:"text"}} - typemod:-1 location:93} is_local:true location:89}} oncommit:ONCOMMIT_NOOP}} stmt_len:99 - here, relpersistence is the information about the persistence of this table where u-> unlogged, p->persistent, t->temporary tables - */ - if createTableNode.CreateStmt.Relation.GetRelpersistence() == "u" { - reportCase(fpath, ISSUE_UNLOGGED_TABLE, "https://github.com/yugabyte/yugabyte-db/issues/1129/", - "Remove UNLOGGED keyword to make it work", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, - UNSUPPORTED_FEATURES, UNLOGGED_TABLE_DOC_LINK) - } -} - -// Checks Whether there is a GIN index -/* -Following type of SQL queries are being taken care of by this function - - 1. CREATE INDEX index_name ON table_name USING gin(column1, column2 ...) - 2. CREATE INDEX index_name ON table_name USING gin(column1 [ASC/DESC/HASH]) -*/ -func checkGinVariations(createIndexNode *pg_query.Node_IndexStmt, sqlStmtInfo sqlInfo, fpath string) { - indexName := createIndexNode.IndexStmt.GetIdxname() - relName := createIndexNode.IndexStmt.GetRelation() - schemaName := relName.GetSchemaname() - tableName := relName.GetRelname() - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - displayObjectName := fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName) - if createIndexNode.IndexStmt.GetAccessMethod() != "gin" { // its always in lower - return - } else { - summaryMap["INDEX"].details[GIN_INDEX_DETAILS] = true - } - /* - e.g. CREATE INDEX idx_name ON public.test USING gin (data, data2); - stmt:{index_stmt:{idxname:"idx_name" relation:{schemaname:"public" relname:"test" inh:true relpersistence:"p" - location:125} access_method:"gin" index_params:{index_elem:{name:"data" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} - index_params:{index_elem:{name:"data2" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}}}} stmt_location:81 stmt_len:81 - */ - if len(createIndexNode.IndexStmt.GetIndexParams()) > 1 { - summaryMap["INDEX"].invalidCount[displayObjectName] = true - reportCase(fpath, "Schema contains gin index on multi column which is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10652", "", "INDEX", displayObjectName, - sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GIN_INDEX_MULTI_COLUMN_DOC_LINK) - return - } - /* - e.g. CREATE INDEX idx_name ON public.test USING gin (data DESC); - stmt:{index_stmt:{idxname:"idx_name" relation:{schemaname:"public" relname:"test" inh:true relpersistence:"p" location:44} - access_method:"gin" index_params:{index_elem:{name:"data" ordering:SORTBY_DESC nulls_ordering:SORTBY_NULLS_DEFAULT}}}} stmt_len:80 - */ - idxParam := createIndexNode.IndexStmt.GetIndexParams()[0] // taking only the first as already checking len > 1 above so should be fine - if idxParam.GetIndexElem().GetOrdering() != pg_query.SortByDir_SORTBY_DEFAULT { - summaryMap["INDEX"].invalidCount[displayObjectName] = true - reportCase(fpath, "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10653", "", "INDEX", displayObjectName, - sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GIN_INDEX_DIFFERENT_ISSUE_DOC_LINK) - } - -} - -func reportPolicyRequireRolesOrGrants(createPolicyNode *pg_query.Node_CreatePolicyStmt, sqlStmtInfo sqlInfo, fpath string) { - policyName := createPolicyNode.CreatePolicyStmt.GetPolicyName() - roles := createPolicyNode.CreatePolicyStmt.GetRoles() - relname := createPolicyNode.CreatePolicyStmt.GetTable() - schemaName := relname.Schemaname - tableName := relname.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - roleNames := make([]string, 0) - /* - e.g. CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); - stmt:{create_policy_stmt:{policy_name:"p" table:{relname:"tbl1" inh:true relpersistence:"p" location:20} cmd_name:"all" - permissive:true roles:{role_spec:{roletype:ROLESPEC_CSTRING rolename:"regress_rls_eve" location:28}} roles:{role_spec: - {roletype:ROLESPEC_CSTRING rolename:"regress_rls_frank" location:45}} qual:{a_const:{boolval:{boolval:true} location:70}}}} - stmt_len:75 - - here role_spec of each roles is managing the roles related information in a POLICY DDL if any, so we can just check if there is - a role name available in it which means there is a role associated with this DDL. Hence report it. - - */ - for _, role := range roles { - roleName := role.GetRoleSpec().GetRolename() // only in case there is role associated with a policy it will error out in schema migration - if roleName != "" { - //this means there is some role or grants used in this Policy, so detecting it - roleNames = append(roleNames, roleName) + err = parserIssueDetector.ParseRequiredDDLs(sqlStmtInfo.formattedStmt) + if err != nil { + utils.ErrExit("error parsing stmt[%s]: %v", sqlStmtInfo.formattedStmt, err) } - } - if len(roleNames) > 0 { - policyNameWithTable := fmt.Sprintf("%s ON %s", policyName, fullyQualifiedName) - summaryMap["POLICY"].invalidCount[policyNameWithTable] = true - reportCase(fpath, fmt.Sprintf("%s Users - (%s)", POLICY_ROLE_ISSUE, strings.Join(roleNames, ",")), "https://github.com/yugabyte/yb-voyager/issues/1655", - "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", - "POLICY", policyNameWithTable, sqlStmtInfo.formattedStmt, MIGRATION_CAVEATS, POLICY_DOC_LINK) - } -} - -func reportUnsupportedDatatypes(relation *pg_query.RangeVar, columns []*pg_query.Node, sqlStmtInfo sqlInfo, fpath string, objectType string) { - schemaName := relation.Schemaname - tableName := relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - for _, column := range columns { - /* - e.g. CREATE TABLE test_xml_type(id int, data xml); - relation:{relname:"test_xml_type" inh:true relpersistence:"p" location:15} table_elts:{column_def:{colname:"id" - type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:32} - is_local:true location:29}} table_elts:{column_def:{colname:"data" type_name:{names:{string:{sval:"xml"}} - typemod:-1 location:42} is_local:true location:37}} oncommit:ONCOMMIT_NOOP}} - - here checking the type of each column as type definition can be a list names for types which are native e.g. int - it has type names - [pg_catalog, int4] both to determine but for complex types like text,json or xml etc. if doesn't have - info about pg_catalog. so checking the 0th only in case XML/XID to determine the type and report - */ - if column.GetColumnDef() != nil { - typeNames := column.GetColumnDef().GetTypeName().GetNames() - typeName, typeSchemaName := getTypeNameAndSchema(typeNames) - fullTypeName := lo.Ternary(typeSchemaName != "", typeSchemaName+"."+typeName, typeName) - isArrayType := len(column.GetColumnDef().GetTypeName().GetArrayBounds()) > 0 - colName := column.GetColumnDef().GetColname() - - liveUnsupportedDatatypes := srcdb.GetPGLiveMigrationUnsupportedDatatypes() - liveWithFfOrFbUnsupportedDatatypes := srcdb.GetPGLiveMigrationWithFFOrFBUnsupportedDatatypes() - - isUnsupportedDatatype := utils.ContainsAnyStringFromSlice(srcdb.PostgresUnsupportedDataTypes, typeName) - isUnsupportedDatatypeInLive := utils.ContainsAnyStringFromSlice(liveUnsupportedDatatypes, typeName) - - isUnsupportedDatatypeInLiveWithFFOrFBList := utils.ContainsAnyStringFromSlice(liveWithFfOrFbUnsupportedDatatypes, typeName) - isUDTDatatype := utils.ContainsAnyStringFromSlice(compositeTypes, fullTypeName) //if type is array - isEnumDatatype := utils.ContainsAnyStringFromSlice(enumTypes, fullTypeName) //is ENUM type - isArrayOfEnumsDatatype := isArrayType && isEnumDatatype - isUnsupportedDatatypeInLiveWithFFOrFB := isUnsupportedDatatypeInLiveWithFFOrFBList || isUDTDatatype || isArrayOfEnumsDatatype - - if isUnsupportedDatatype { - reason := fmt.Sprintf("%s - %s on column - %s", UNSUPPORTED_DATATYPE, typeName, colName) - summaryMap[objectType].invalidCount[sqlStmtInfo.objName] = true - var ghIssue, suggestion, docLink string - - switch typeName { - case "xml": - ghIssue = "https://github.com/yugabyte/yugabyte-db/issues/1043" - suggestion = "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details." - docLink = XML_DATATYPE_DOC_LINK - case "xid": - ghIssue = "https://github.com/yugabyte/yugabyte-db/issues/15638" - suggestion = "Functions for this type e.g. txid_current are not supported in YugabyteDB yet" - docLink = XID_DATATYPE_DOC_LINK - case "geometry", "geography", "box2d", "box3d", "topogeometry": - ghIssue = "https://github.com/yugabyte/yugabyte-db/issues/11323" - suggestion = "" - docLink = UNSUPPORTED_DATATYPES_DOC_LINK - default: - ghIssue = "https://github.com/yugabyte/yb-voyager/issues/1731" - suggestion = "" - docLink = UNSUPPORTED_DATATYPES_DOC_LINK - } - reportCase(fpath, reason, ghIssue, suggestion, - objectType, fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_DATATYPES, docLink) - } else if objectType == TABLE && isUnsupportedDatatypeInLive { - //reporting only for TABLE Type as we don't deal with FOREIGN TABLE in live migration - reason := fmt.Sprintf("%s - %s on column - %s", UNSUPPORTED_DATATYPE_LIVE_MIGRATION, typeName, colName) - summaryMap[objectType].invalidCount[sqlStmtInfo.objName] = true - reportCase(fpath, reason, "https://github.com/yugabyte/yb-voyager/issues/1731", "", - objectType, fullyQualifiedName, sqlStmtInfo.formattedStmt, MIGRATION_CAVEATS, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK) - } else if objectType == TABLE && isUnsupportedDatatypeInLiveWithFFOrFB { - //reporting only for TABLE Type as we don't deal with FOREIGN TABLE in live migration - reportTypeName := fullTypeName - if isArrayType { // For Array cases to make it clear in issue - reportTypeName = fmt.Sprintf("%s[]", reportTypeName) - } - //reporting types in the list YugabyteUnsupportedDataTypesForDbzm, UDT columns as unsupported with live migration with ff/fb - reason := fmt.Sprintf("%s - %s on column - %s", UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB, reportTypeName, colName) - summaryMap[objectType].invalidCount[sqlStmtInfo.objName] = true - reportCase(fpath, reason, "https://github.com/yugabyte/yb-voyager/issues/1731", "", - objectType, fullyQualifiedName, sqlStmtInfo.formattedStmt, MIGRATION_CAVEATS, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK) - } + if parserIssueDetector.IsGinIndexPresentInSchema { + summaryMap["INDEX"].details[GIN_INDEX_DETAILS] = true } - } -} -func getTypeNameAndSchema(typeNames []*pg_query.Node) (string, string) { - typeName := "" - typeSchemaName := "" - if len(typeNames) > 0 { - typeName = typeNames[len(typeNames)-1].GetString_().Sval // type name can be qualified / unqualifed or native / non-native proper type name will always be available at last index - } - if len(typeNames) >= 2 { // Names list will have all the parts of qualified type name - typeSchemaName = typeNames[len(typeNames)-2].GetString_().Sval // // type name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index - } - - return typeName, typeSchemaName -} - -var deferrableConstraintsList = []pg_query.ConstrType{ - pg_query.ConstrType_CONSTR_ATTR_DEFERRABLE, - pg_query.ConstrType_CONSTR_ATTR_DEFERRED, - pg_query.ConstrType_CONSTR_ATTR_IMMEDIATE, -} - -func reportDeferrableConstraintCreateTable(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - columns := createTableNode.CreateStmt.TableElts - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - - for _, column := range columns { - /* - e.g. create table unique_def_test(id int UNIQUE DEFERRABLE, c1 int); - create_stmt:{relation:{relname:"unique_def_test" inh:true relpersistence:"p" location:15} - table_elts:{column_def:{colname:"id" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} - typemod:-1 location:34} is_local:true constraints:{constraint:{contype:CONSTR_UNIQUE location:38}} - constraints:{constraint:{contype:CONSTR_ATTR_DEFERRABLE location:45}} location:31}} .... - - here checking the case where this clause is in column definition so iterating over each column_def and in that - constraint type has deferrable or not and also it should not be a foreign constraint as Deferrable on FKs are - supported. - */ - if column.GetColumnDef() != nil { - constraints := column.GetColumnDef().GetConstraints() - colName := column.GetColumnDef().GetColname() - if constraints != nil { - isDeferrable := false - var deferrableConstraintType pg_query.ConstrType - for idx, constraint := range constraints { - if slices.Contains(deferrableConstraintsList, constraint.GetConstraint().Contype) { - //Getting the constraint type before the DEFERRABLE clause as the clause is applicable to that constraint - if idx > 0 { - deferrableConstraintType = constraints[idx-1].GetConstraint().Contype - } - isDeferrable = true - } - } - if isDeferrable && deferrableConstraintType != pg_query.ConstrType_CONSTR_FOREIGN { - summaryMap["TABLE"].invalidCount[sqlStmtInfo.objName] = true - generatedConName := generateConstraintName(deferrableConstraintType, tableName, []string{colName}) - specifiedConstraintName := column.GetConstraint().GetConname() - conName := lo.Ternary(specifiedConstraintName == "", generatedConName, specifiedConstraintName) - reportCase(fpath, DEFERRABLE_CONSTRAINT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/1709", - "Remove these constraints from the exported schema and make the necessary changes to the application before pointing it to target", - "TABLE", fmt.Sprintf("%s, constraint: (%s)", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, DEFERRABLE_CONSTRAINT_DOC_LINK) - } - } - } else if column.GetConstraint() != nil { - /* - e.g. create table uniquen_def_test1(id int, c1 int, UNIQUE(id) DEFERRABLE INITIALLY DEFERRED); - {create_stmt:{relation:{relname:"unique_def_test1" inh:true relpersistence:"p" location:80} table_elts:{column_def:{colname:"id" - type_name:{.... names:{string:{sval:"int4"}} typemod:-1 location:108} is_local:true location:105}} - table_elts:{constraint:{contype:CONSTR_UNIQUE deferrable:true initdeferred:true location:113 keys:{string:{sval:"id"}}}} .. - - here checking the case where this constraint is at the at the end as a constraint only, so checking deferrable field in constraint - in case of its not a FK. - */ - colNames := getColumnNames(column.GetConstraint().GetKeys()) - if column.GetConstraint().Deferrable && column.GetConstraint().Contype != pg_query.ConstrType_CONSTR_FOREIGN { - generatedConName := generateConstraintName(column.GetConstraint().Contype, tableName, colNames) - specifiedConstraintName := column.GetConstraint().GetConname() - conName := lo.Ternary(specifiedConstraintName == "", generatedConName, specifiedConstraintName) - summaryMap["TABLE"].invalidCount[sqlStmtInfo.objName] = true - reportCase(fpath, DEFERRABLE_CONSTRAINT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/1709", - "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", - "TABLE", fmt.Sprintf("%s, constraint: (%s)", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, DEFERRABLE_CONSTRAINT_DOC_LINK) - } - } - } -} - -func generateConstraintName(conType pg_query.ConstrType, tableName string, columns []string) string { - suffix := "" - //Deferrable is only applicable to following constraint - //https://www.postgresql.org/docs/current/sql-createtable.html#:~:text=Currently%2C%20only%20UNIQUE%2C%20PRIMARY%20KEY%2C%20EXCLUDE%2C%20and%20REFERENCES - switch conType { - case pg_query.ConstrType_CONSTR_UNIQUE: - suffix = "_key" - case pg_query.ConstrType_CONSTR_PRIMARY: - suffix = "_pkey" - case pg_query.ConstrType_CONSTR_EXCLUSION: - suffix = "_excl" - case pg_query.ConstrType_CONSTR_FOREIGN: - suffix = "_fkey" - } - - return fmt.Sprintf("%s_%s%s", tableName, strings.Join(columns, "_"), suffix) -} - -func getColumnNames(keys []*pg_query.Node) []string { - var res []string - for _, k := range keys { - res = append(res, k.GetString_().Sval) - } - return res -} - -func reportDeferrableConstraintAlterTable(alterTableNode *pg_query.Node_AlterTableStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := alterTableNode.AlterTableStmt.Relation.Schemaname - tableName := alterTableNode.AlterTableStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - - alterCmd := alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd() - /* - e.g. ALTER TABLE ONLY public.users ADD CONSTRAINT users_email_key UNIQUE (email) DEFERRABLE; - alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_UNIQUE conname:"users_email_key" - deferrable:true location:196 keys:{string:{sval:"email"}}}} behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} - - similar to CREATE table 2nd case where constraint is at the end of column definitions mentioning the constraint only - so here as well while adding constraint checking the type of constraint and the deferrable field of it. - */ - constraint := alterCmd.GetDef().GetConstraint() - if constraint != nil && constraint.Deferrable && constraint.Contype != pg_query.ConstrType_CONSTR_FOREIGN { - conName := constraint.Conname - summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true - reportCase(fpath, DEFERRABLE_CONSTRAINT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/1709", - "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", - "TABLE", fmt.Sprintf("%s, constraint: (%s)", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, DEFERRABLE_CONSTRAINT_DOC_LINK) - } -} - -func reportExclusionConstraintCreateTable(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { - - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - columns := createTableNode.CreateStmt.TableElts - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - /* - e.g. CREATE TABLE "Test"( - id int, - room_id int, - time_range tsrange, - room_id1 int, - time_range1 tsrange - EXCLUDE USING gist (room_id WITH =, time_range WITH &&), - EXCLUDE USING gist (room_id1 WITH =, time_range1 WITH &&) - ); - create_stmt:{relation:{relname:"Test" inh:true relpersistence:"p" location:14} table_elts:...table_elts:{constraint:{contype:CONSTR_EXCLUSION - location:226 exclusions:{list:{items:{index_elem:{name:"room_id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} - items:{list:{items:{string:{sval:"="}}}}}} exclusions:{list:{items:{index_elem:{name:"time_range" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} - items:{list:{items:{string:{sval:"&&"}}}}}} access_method:"gist"}} table_elts:{constraint:{contype:CONSTR_EXCLUSION location:282 exclusions:{list: - {items:{index_elem:{name:"room_id1" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} items:{list:{items:{string:{sval:"="}}}}}} - exclusions:{list:{items:{index_elem:{name:"time_range1" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} items:{list:{items:{string:{sval:"&&"}}}}}} - access_method:"gist"}} oncommit:ONCOMMIT_NOOP}} stmt_len:365} - here we are iterating over all the table_elts - table elements and which are comma separated column info in - the DDL so each column has column_def(column definition) in the parse tree but in case it is a constraint, the column_def - is nil. - - */ - for _, column := range columns { - //In case CREATE DDL has EXCLUDE USING gist(room_id '=', time_range WITH &&) - it will be included in columns but won't have columnDef as its a constraint - if column.GetColumnDef() == nil && column.GetConstraint() != nil { - if column.GetConstraint().Contype == pg_query.ConstrType_CONSTR_EXCLUSION { - colNames := getColumnNamesFromExclusions(column.GetConstraint().GetExclusions()) - generatedConName := generateConstraintName(column.GetConstraint().Contype, tableName, colNames) - specifiedConstraintName := column.GetConstraint().GetConname() - conName := lo.Ternary(specifiedConstraintName == "", generatedConName, specifiedConstraintName) - summaryMap["TABLE"].invalidCount[sqlStmtInfo.objName] = true - reportCase(fpath, EXCLUSION_CONSTRAINT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/3944", - "Refer docs link for details on possible workaround", "TABLE", fmt.Sprintf("%s, constraint: (%s)", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, EXCLUSION_CONSTRAINT_DOC_LINK) - } + ddlIssues, err := parserIssueDetector.GetDDLIssues(sqlStmtInfo.formattedStmt, targetDbVersion) + if err != nil { + utils.ErrExit("error getting ddl issues for stmt[%s]: %v", sqlStmtInfo.formattedStmt, err) } - } -} - -func getColumnNamesFromExclusions(keys []*pg_query.Node) []string { - var res []string - for _, k := range keys { - //exclusions:{list:{items:{index_elem:{name:"room_id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} - //items:{list:{items:{string:{sval:"="}}}}}} - res = append(res, k.GetList().GetItems()[0].GetIndexElem().Name) // every first element of items in exclusions will be col name - } - return res -} - -func reportCreateIndexStorageParameter(createIndexNode *pg_query.Node_IndexStmt, sqlStmtInfo sqlInfo, fpath string) { - indexName := createIndexNode.IndexStmt.GetIdxname() - relName := createIndexNode.IndexStmt.GetRelation() - schemaName := relName.GetSchemaname() - tableName := relName.GetRelname() - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - /* - e.g. CREATE INDEX idx on table_name(id) with (fillfactor='70'); - index_stmt:{idxname:"idx" relation:{relname:"table_name" inh:true relpersistence:"p" location:21} access_method:"btree" - index_params:{index_elem:{name:"id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} - options:{def_elem:{defname:"fillfactor" arg:{string:{sval:"70"}} ... - here again similar to ALTER table Storage parameters options is the high level field in for WITH options. - */ - if len(createIndexNode.IndexStmt.GetOptions()) > 0 { - //YB doesn't support any storage parameters from PG yet refer - - //https://docs.yugabyte.com/preview/api/ysql/the-sql-language/statements/ddl_create_table/#storage-parameters-1 - summaryMap["INDEX"].invalidCount[fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName)] = true - reportCase(fpath, STORAGE_PARAMETERS_DDL_STMT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/23467", - "Remove the storage parameters from the DDL", "INDEX", indexName, sqlStmtInfo.stmt, UNSUPPORTED_FEATURES, STORAGE_PARAMETERS_DDL_STMT_DOC_LINK) - } -} - -func reportAlterTableVariants(alterTableNode *pg_query.Node_AlterTableStmt, sqlStmtInfo sqlInfo, fpath string, objType string) { - schemaName := alterTableNode.AlterTableStmt.Relation.Schemaname - tableName := alterTableNode.AlterTableStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - // this will the list of items in the SET (attribute=value, ..) - /* - e.g. alter table test_1 alter column col1 set (attribute_option=value); - cmds:{alter_table_cmd:{subtype:AT_SetOptions name:"col1" def:{list:{items:{def_elem:{defname:"attribute_option" - arg:{type_name:{names:{string:{sval:"value"}} typemod:-1 location:263}} defaction:DEFELEM_UNSPEC location:246}}}}... - for set attribute issue we will the type of alter setting the options and in the 'def' definition field which has the - information of the type, we will check if there is any list which will only present in case there is syntax like (...) - */ - if alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetSubtype() == pg_query.AlterTableType_AT_SetOptions && - len(alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetDef().GetList().GetItems()) > 0 { - reportCase(fpath, ALTER_TABLE_SET_ATTRIBUTE_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/1124", - "Remove it from the exported schema", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, UNSUPPORTED_ALTER_VARIANTS_DOC_LINK) - } - - /* - e.g. alter table test add constraint uk unique(id) with (fillfactor='70'); - alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_UNIQUE conname:"asd" location:292 - keys:{string:{sval:"id"}} options:{def_elem:{defname:"fillfactor" arg:{string:{sval:"70"}}... - Similarly here we are trying to get the constraint if any and then get the options field which is WITH options - in this case only so checking that for this case. - */ - - if alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetSubtype() == pg_query.AlterTableType_AT_AddConstraint && - len(alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetDef().GetConstraint().GetOptions()) > 0 { - reportCase(fpath, STORAGE_PARAMETERS_DDL_STMT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/23467", - "Remove the storage parameters from the DDL", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, STORAGE_PARAMETERS_DDL_STMT_DOC_LINK) - } - - /* - e.g. ALTER TABLE example DISABLE example_rule; - cmds:{alter_table_cmd:{subtype:AT_DisableRule name:"example_rule" behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} - checking the subType is sufficient in this case - */ - if alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetSubtype() == pg_query.AlterTableType_AT_DisableRule { - ruleName := alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetName() - reportCase(fpath, ALTER_TABLE_DISABLE_RULE_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/1124", - fmt.Sprintf("Remove this and the rule '%s' from the exported schema to be not enabled on the table.", ruleName), "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, UNSUPPORTED_ALTER_VARIANTS_DOC_LINK) - } - /* - e.g. ALTER TABLE example CLUSTER ON idx; - stmt:{alter_table_stmt:{relation:{relname:"example" inh:true relpersistence:"p" location:13} - cmds:{alter_table_cmd:{subtype:AT_ClusterOn name:"idx" behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} stmt_len:32 - - */ - if alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetSubtype() == pg_query.AlterTableType_AT_ClusterOn { - reportCase(fpath, ALTER_TABLE_CLUSTER_ON_ISSUE, - "https://github.com/YugaByte/yugabyte-db/issues/1124", "Remove it from the exported schema.", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, UNSUPPORTED_ALTER_VARIANTS_DOC_LINK) - } - -} - -func reportExclusionConstraintAlterTable(alterTableNode *pg_query.Node_AlterTableStmt, sqlStmtInfo sqlInfo, fpath string) { - - schemaName := alterTableNode.AlterTableStmt.Relation.Schemaname - tableName := alterTableNode.AlterTableStmt.Relation.Relname - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - alterCmd := alterTableNode.AlterTableStmt.Cmds[0].GetAlterTableCmd() - /* - e.g. ALTER TABLE ONLY public.meeting ADD CONSTRAINT no_time_overlap EXCLUDE USING gist (room_id WITH =, time_range WITH &&); - cmds:{alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_EXCLUSION conname:"no_time_overlap" location:41 - here again same checking the definition of the alter stmt if it has constraint and checking its type - */ - constraint := alterCmd.GetDef().GetConstraint() - if alterCmd.Subtype == pg_query.AlterTableType_AT_AddConstraint && constraint.Contype == pg_query.ConstrType_CONSTR_EXCLUSION { - // colNames := getColumnNamesFromExclusions(alterCmd.GetDef().GetConstraint().GetExclusions()) - conName := constraint.Conname - summaryMap["TABLE"].invalidCount[fullyQualifiedName] = true - reportCase(fpath, EXCLUSION_CONSTRAINT_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/3944", - "Refer docs link for details on possible workaround", "TABLE", fmt.Sprintf("%s, constraint: (%s)", fullyQualifiedName, conName), sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, EXCLUSION_CONSTRAINT_DOC_LINK) - } -} - -func reportGeneratedStoredColumnTables(createTableNode *pg_query.Node_CreateStmt, sqlStmtInfo sqlInfo, fpath string) { - schemaName := createTableNode.CreateStmt.Relation.Schemaname - tableName := createTableNode.CreateStmt.Relation.Relname - columns := createTableNode.CreateStmt.TableElts - var generatedColumns []string - for _, column := range columns { - //In case CREATE DDL has PRIMARY KEY(column_name) - it will be included in columns but won't have columnDef as its a constraint - if column.GetColumnDef() != nil { - constraints := column.GetColumnDef().Constraints - for _, constraint := range constraints { - if constraint.GetConstraint().Contype == pg_query.ConstrType_CONSTR_GENERATED { - generatedColumns = append(generatedColumns, column.GetColumnDef().Colname) - } - } + for _, i := range ddlIssues { + schemaAnalysisReport.Issues = append(schemaAnalysisReport.Issues, convertIssueInstanceToAnalyzeIssue(i, fpath, false)) } } - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) - if len(generatedColumns) > 0 { - summaryMap["TABLE"].invalidCount[sqlStmtInfo.objName] = true - reportCase(fpath, STORED_GENERATED_COLUMN_ISSUE_REASON+fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")), - "https://github.com/yugabyte/yugabyte-db/issues/10695", - "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", - TABLE, fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GENERATED_STORED_COLUMN_DOC_LINK) - } } // Checks compatibility of views @@ -1558,10 +445,6 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE clause not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") - } else if tbl := inheritRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { - summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, INHERITANCE_ISSUE_REASON, - "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, INHERITANCE_DOC_LINK) } else if tbl := withOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "OIDs are not supported for user tables.", @@ -1693,37 +576,64 @@ func checker(sqlInfoArr []sqlInfo, fpath string, objType string) { checkDDL(sqlInfoArr, fpath, objType) checkForeign(sqlInfoArr, fpath) checkRemaining(sqlInfoArr, fpath) + checkStmtsUsingParser(sqlInfoArr, fpath, objType) if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { checkPlPgSQLStmtsUsingParser(sqlInfoArr, fpath, objType) } - checkStmtsUsingParser(sqlInfoArr, fpath, objType) } func checkPlPgSQLStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { for _, sqlInfoStmt := range sqlInfoArr { - issues, err := parserIssueDetector.GetAllIssues(sqlInfoStmt.formattedStmt, targetDbVersion) + issues, err := parserIssueDetector.GetAllPLPGSQLIssues(sqlInfoStmt.formattedStmt, targetDbVersion) if err != nil { log.Infof("error in getting the issues-%s: %v", sqlInfoStmt.formattedStmt, err) continue } for _, issueInstance := range issues { - issue := convertIssueInstanceToAnalyzeIssue(issueInstance, fpath) + issue := convertIssueInstanceToAnalyzeIssue(issueInstance, fpath, true) schemaAnalysisReport.Issues = append(schemaAnalysisReport.Issues, issue) } } } -func convertIssueInstanceToAnalyzeIssue(issueInstance issue.IssueInstance, fileName string) utils.Issue { - summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true +var MigrationCaveatsIssues = []string{ + ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, + FOREIGN_TABLE_ISSUE_REASON, + POLICY_ROLE_ISSUE, + UNSUPPORTED_DATATYPE_LIVE_MIGRATION, + UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB, +} + +func convertIssueInstanceToAnalyzeIssue(issueInstance issue.IssueInstance, fileName string, isPlPgSQLIssue bool) utils.Issue { + issueType := UNSUPPORTED_FEATURES + switch true { + case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { + //Adding the MIGRATION_CAVEATS issueType of the utils.Issue for these issueInstances in MigrationCaveatsIssues + return strings.Contains(issueInstance.TypeName, i) + }): + issueType = MIGRATION_CAVEATS + case strings.HasPrefix(issueInstance.TypeName, UNSUPPORTED_DATATYPE): + //Adding the UNSUPPORTED_DATATYPES issueType of the utils.Issue for these issues whose TypeName starts with "Unsupported datatype ..." + issueType = UNSUPPORTED_DATATYPES + case isPlPgSQLIssue: + issueType = UNSUPPORTED_PLPGSQL_OBEJCTS + } + + //TODO: how to different between same issue on differnt obejct types like ALTER/INDEX for not adding it ot invalid count map + increaseInvalidCount, ok := issueInstance.Details["INCREASE_INVALID_COUNT"] + if !ok || (increaseInvalidCount.(bool)) { + summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true + } + return utils.Issue{ ObjectType: issueInstance.ObjectType, ObjectName: issueInstance.ObjectName, Reason: issueInstance.TypeName, - SqlStatement: issueInstance.SqlStatement, //Displaying the actual query in the PLPGSQL block that is problematic + SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, - IssueType: UNSUPPORTED_PLPGSQL_OBEJCTS, + IssueType: issueType, Suggestion: issueInstance.Suggestion, GH: issueInstance.GH, } @@ -2111,9 +1021,6 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source, detectIssues bool) utils. if objType == "EXTENSION" { checkExtensions(sqlInfoArr, filePath) } - if objType == "FOREIGN TABLE" { - checkForeignTable(sqlInfoArr, filePath) - } checker(sqlInfoArr, filePath, objType) if objType == "CONVERSION" { diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index b4cb76bb98..f15c43a0f7 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -967,7 +967,7 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for PG...") unsupportedFeatures := make([]UnsupportedFeature, 0) - for _, indexMethod := range unsupportedIndexMethods { + for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) feature := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) @@ -986,7 +986,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, EXCLUSION_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, UnsupportedIndexDatatypes)) + unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)) pkOrUkConstraintIssuePrefix := strings.Split(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, "%s")[0] unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, schemaAnalysisReport, false, "")) @@ -1127,7 +1127,6 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error if source.DBType != POSTGRESQL { return nil, nil } - parserIssueDetector := queryissue.NewParserIssueDetector() query := fmt.Sprintf("SELECT DISTINCT query from %s", migassessment.DB_QUERIES_SUMMARY) rows, err := assessmentDB.Query(query) if err != nil { @@ -1250,9 +1249,9 @@ func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []ut isUnsupportedDatatypeInLive := utils.ContainsAnyStringFromSlice(liveUnsupportedDatatypes, typeName) isUnsupportedDatatypeInLiveWithFFOrFBList := utils.ContainsAnyStringFromSlice(liveWithFForFBUnsupportedDatatypes, typeName) - isUDTDatatype := utils.ContainsAnyStringFromSlice(compositeTypes, allColumnsDataTypes[i].DataType) - isArrayDatatype := strings.HasSuffix(allColumnsDataTypes[i].DataType, "[]") //if type is array - isEnumDatatype := utils.ContainsAnyStringFromSlice(enumTypes, strings.TrimSuffix(allColumnsDataTypes[i].DataType, "[]")) //is ENUM type + isUDTDatatype := utils.ContainsAnyStringFromSlice(parserIssueDetector.GetCompositeTypes(), allColumnsDataTypes[i].DataType) + isArrayDatatype := strings.HasSuffix(allColumnsDataTypes[i].DataType, "[]") //if type is array + isEnumDatatype := utils.ContainsAnyStringFromSlice(parserIssueDetector.GetEnumTypes(), strings.TrimSuffix(allColumnsDataTypes[i].DataType, "[]")) //is ENUM type isArrayOfEnumsDatatype := isArrayDatatype && isEnumDatatype isUnsupportedDatatypeInLiveWithFFOrFB := isUnsupportedDatatypeInLiveWithFFOrFBList || isUDTDatatype || isArrayOfEnumsDatatype diff --git a/yb-voyager/src/issue/constants.go b/yb-voyager/src/issue/constants.go index f792d32fea..6599de0fae 100644 --- a/yb-voyager/src/issue/constants.go +++ b/yb-voyager/src/issue/constants.go @@ -18,15 +18,49 @@ package issue // Types const ( - ADVISORY_LOCKS = "ADVISORY_LOCKS" - SYSTEM_COLUMNS = "SYSTEM_COLUMNS" - XML_FUNCTIONS = "XML_FUNCTIONS" - REFERENCED_TYPE_DECLARATION = "REFERENCED_TYPE_DECLARATION" + ADVISORY_LOCKS = "ADVISORY_LOCKS" + SYSTEM_COLUMNS = "SYSTEM_COLUMNS" + XML_FUNCTIONS = "XML_FUNCTIONS" + + REFERENCED_TYPE_DECLARATION = "REFERENCED_TYPE_DECLARATION" + STORED_GENERATED_COLUMNS = "STORED_GENERATED_COLUMNS" + UNLOGGED_TABLE = "UNLOGGED_TABLE" + UNSUPPORTED_INDEX_METHOD = "UNSUPPORTED_INDEX_METHOD" + STORAGE_PARAMETER = "STORAGE_PARAMETER" + SET_ATTRIBUTES = "SET_ATTRIBUTES" + CLUSTER_ON = "CLUSTER_ON" + DISABLE_RULE = "DISABLE_RULE" + EXCLUSION_CONSTRAINTS = "EXCLUSION_CONSTRAINTS" + DEFERRABLE_CONSTRAINTS = "DEFERRABLE_CONSTRAINTS" + MULTI_COLUMN_GIN_INDEX = "MULTI_COLUMN_GIN_INDEX" + ORDERED_GIN_INDEX = "ORDERED_GIN_INDEX" + POLICY_WITH_ROLES = "POLICY_WITH_ROLES" + CONSTRAINT_TRIGGER = "CONSTRAINT_TRIGGER" + REFERENCING_CLAUSE_FOR_TRIGGERS = "REFERENCING_CLAUSE_FOR_TRIGGERS" + BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE = "BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE" + ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE = "ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE" + EXPRESSION_PARTITION_WITH_PK_UK = "EXPRESSION_PARTITION_WITH_PK_UK" + MULTI_COLUMN_LIST_PARTITION = "MULTI_COLUMN_LIST_PARTITION" + INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION = "INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION" + XML_DATATYPE = "XML_DATATYPE" + XID_DATATYPE = "XID_DATATYPE" + POSTGIS_DATATYPES = "POSTGIS_DATATYPES" + UNSUPPORTED_DATATYPES = "UNSUPPORTED_TYPES" + UNSUPPORTED_DATATYPES_LIVE_MIGRATION = "UNSUPPORTED_DATATYPES_LIVE_MIGRATION" + UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB = "UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB" + PK_UK_ON_COMPLEX_DATATYPE = "PK_UK_ON_COMPLEX_DATATYPE" + INDEX_ON_COMPLEX_DATATYPE = "INDEX_ON_COMPLEX_DATATYPE" + FOREIGN_TABLE = "FOREIGN_TABLE" + INHERITANCE = "INHERITANCE" ) // Object types const ( - TABLE_OBJECT_TYPE = "TABLE" - FUNCTION_OBJECT_TYPE = "FUNCTION" - DML_QUERY_OBJECT_TYPE = "DML_QUERY" + TABLE_OBJECT_TYPE = "TABLE" + FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" + FUNCTION_OBJECT_TYPE = "FUNCTION" + INDEX_OBJECT_TYPE = "INDEX" + POLICY_OBJECT_TYPE = "POLICY" + TRIGGER_OBJECT_TYPE = "TRIGGER" + DML_QUERY_OBJECT_TYPE = "DML_QUERY" ) diff --git a/yb-voyager/src/issue/ddl.go b/yb-voyager/src/issue/ddl.go index 390ee1b76c..74c5573f89 100644 --- a/yb-voyager/src/issue/ddl.go +++ b/yb-voyager/src/issue/ddl.go @@ -16,6 +16,422 @@ limitations under the License. package issue +import ( + "fmt" + "strings" +) + +const ( + DOCS_LINK_PREFIX = "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/" + POSTGRESQL_PREFIX = "postgresql/" + MYSQL_PREFIX = "mysql/" + ORACLE_PREFIX = "oracle/" +) + +var generatedColumnsIssue = Issue{ + Type: STORED_GENERATED_COLUMNS, + TypeName: "Stored generated columns are not supported.", + GH: "https://github.com/yugabyte/yugabyte-db/issues/10695", + Suggestion: "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#generated-always-as-stored-type-column-is-not-supported", +} + +func NewGeneratedColumnsIssue(objectType string, objectName string, sqlStatement string, generatedColumns []string) IssueInstance { + issue := generatedColumnsIssue + issue.TypeName = issue.TypeName + fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")) + return newIssueInstance(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var unloggedTableIssue = Issue{ + Type: UNLOGGED_TABLE, + TypeName: "UNLOGGED tables are not supported yet.", + GH: "https://github.com/yugabyte/yugabyte-db/issues/1129/", + Suggestion: "Remove UNLOGGED keyword to make it work", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unlogged-table-is-not-supported", +} + +func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + details := map[string]interface{}{} + //for UNLOGGED TABLE as its not reported in the TABLE objects + if objectType == "TABLE" { + details["INCREASE_INVALID_COUNT"] = false + } + return newIssueInstance(unloggedTableIssue, objectType, objectName, sqlStatement, details) +} + +var unsupportedIndexMethodIssue = Issue{ + Type: UNSUPPORTED_INDEX_METHOD, + TypeName: "Schema contains %s index which is not supported.", + GH: "https://github.com/YugaByte/yugabyte-db/issues/1337", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#gist-brin-and-spgist-index-types-are-not-supported", +} + +func NewUnsupportedIndexMethodIssue(objectType string, objectName string, sqlStatement string, indexAccessMethod string) IssueInstance { + issue := unsupportedIndexMethodIssue + issue.TypeName = fmt.Sprintf(unsupportedIndexMethodIssue.TypeName, strings.ToUpper(indexAccessMethod)) + return newIssueInstance(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var storageParameterIssue = Issue{ + Type: STORAGE_PARAMETER, + TypeName: "Storage parameters are not supported yet.", + GH: "https://github.com/yugabyte/yugabyte-db/issues/23467", + Suggestion: "Remove the storage parameters from the DDL", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", +} + +func NewStorageParameterIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + details := map[string]interface{}{} + //for ALTER AND INDEX both same struct now how to differentiate which one to not + if objectType == "TABLE" { + details["INCREASE_INVALID_COUNT"] = false + } + return newIssueInstance(storageParameterIssue, objectType, objectName, sqlStatement, details) +} + +var setAttributeIssue = Issue{ + Type: SET_ATTRIBUTES, + TypeName: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", + GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", + Suggestion: "Remove it from the exported schema", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-alter-table-ddl-variants-in-source-schema", +} + +func NewSetAttributeIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + details := map[string]interface{}{} + //for ALTER AND INDEX both same struct now how to differentiate which one to not + if objectType == "TABLE" { + details["INCREASE_INVALID_COUNT"] = false + } + return newIssueInstance(setAttributeIssue, objectType, objectName, sqlStatement, details) +} + +var clusterOnIssue = Issue{ + Type: CLUSTER_ON, + TypeName: "ALTER TABLE CLUSTER not supported yet.", + GH: "https://github.com/YugaByte/yugabyte-db/issues/1124", + Suggestion: "Remove it from the exported schema.", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-alter-table-ddl-variants-in-source-schema", +} + +func NewClusterONIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + details := map[string]interface{}{} + //for ALTER AND INDEX both same struct now how to differentiate which one to not + if objectType == "TABLE" { + details["INCREASE_INVALID_COUNT"] = false + } + return newIssueInstance(clusterOnIssue, objectType, objectName, sqlStatement, details) +} + +var disableRuleIssue = Issue{ + Type: DISABLE_RULE, + TypeName: "ALTER TABLE name DISABLE RULE not supported yet", + GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", + Suggestion: "Remove this and the rule '%s' from the exported schema to be not enabled on the table.", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-alter-table-ddl-variants-in-source-schema", +} + +func NewDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) IssueInstance { + details := map[string]interface{}{} + //for ALTER AND INDEX both same struct now how to differentiate which one to not + if objectType == "TABLE" { + details["INCREASE_INVALID_COUNT"] = false + } + issue := disableRuleIssue + issue.Suggestion = fmt.Sprintf(issue.Suggestion, ruleName) + return newIssueInstance(issue, objectType, objectName, sqlStatement, details) +} + +var exclusionConstraintIssue = Issue{ + Type: EXCLUSION_CONSTRAINTS, + TypeName: "Exclusion constraint is not supported yet", + GH: "https://github.com/yugabyte/yugabyte-db/issues/3944", + Suggestion: "Refer docs link for details on possible workaround", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#exclusion-constraints-is-not-supported", +} + +func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + return newIssueInstance(exclusionConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var deferrableConstraintIssue = Issue{ + Type: DEFERRABLE_CONSTRAINTS, + TypeName: "DEFERRABLE constraints not supported yet", + GH: "https://github.com/yugabyte/yugabyte-db/issues/1709", + Suggestion: "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", +} + +func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + return newIssueInstance(deferrableConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var multiColumnGinIndexIssue = Issue{ + Type: MULTI_COLUMN_GIN_INDEX, + TypeName: "Schema contains gin index on multi column which is not supported.", + GH: "https://github.com/yugabyte/yugabyte-db/issues/10652", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#gin-indexes-on-multiple-columns-are-not-supported", +} + +func NewMultiColumnGinIndexIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + return newIssueInstance(multiColumnGinIndexIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var orderedGinIndexIssue = Issue{ + Type: ORDERED_GIN_INDEX, + TypeName: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", + GH: "https://github.com/yugabyte/yugabyte-db/issues/10653", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#issue-in-some-unsupported-cases-of-gin-indexes", +} + +func NewOrderedGinIndexIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + return newIssueInstance(orderedGinIndexIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var policyRoleIssue = Issue{ + Type: POLICY_WITH_ROLES, + TypeName: "Policy require roles to be created.", + Suggestion: "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", + GH: "https://github.com/yugabyte/yb-voyager/issues/1655", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#policies-on-users-in-source-require-manual-user-creation", +} + +func NewPolicyRoleIssue(objectType string, objectName string, SqlStatement string, roles []string) IssueInstance { + issue := policyRoleIssue + issue.TypeName = fmt.Sprintf("%s Users - (%s)", issue.TypeName, strings.Join(roles, ",")) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var constraintTriggerIssue = Issue{ + Type: CONSTRAINT_TRIGGER, + TypeName: "CONSTRAINT TRIGGER not supported yet.", + GH: "https://github.com/YugaByte/yugabyte-db/issues/1709", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#constraint-trigger-is-not-supported", +} + +func NewConstraintTriggerIssue(objectType string, objectName string, SqlStatement string) IssueInstance { + details := map[string]interface{}{} + //for CONSTRAINT TRIGGER we don't have separate object type TODO: fix + if objectType == "TRIGGER" { + details["INCREASE_INVALID_COUNT"] = false + } + return newIssueInstance(constraintTriggerIssue, objectType, objectName, SqlStatement, details) +} + +var referencingClauseInTriggerIssue = Issue{ + Type: REFERENCING_CLAUSE_FOR_TRIGGERS, + TypeName: "REFERENCING clause (transition tables) not supported yet.", + GH: "https://github.com/YugaByte/yugabyte-db/issues/1668", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#referencing-clause-for-triggers", +} + +func NewReferencingClauseTrigIssue(objectType string, objectName string, SqlStatement string) IssueInstance { + return newIssueInstance(referencingClauseInTriggerIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var beforeRowTriggerOnPartitionTableIssue = Issue{ + Type: BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, + TypeName: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#before-row-triggers-on-partitioned-tables", + GH: "https://github.com/yugabyte/yugabyte-db/issues/24830", + Suggestion: "Create the triggers on individual partitions.", +} + +func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, SqlStatement string) IssueInstance { + return newIssueInstance(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var alterTableAddPKOnPartitionIssue = Issue{ + Type: ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, + TypeName: "Adding primary key to a partitioned table is not supported yet.", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#adding-primary-key-to-a-partitioned-table-results-in-an-error", + GH: "https://github.com/yugabyte/yugabyte-db/issues/10074", +} + +func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, SqlStatement string) IssueInstance { + details := map[string]interface{}{} + //for ALTER AND INDEX both same struct now how to differentiate which one to not + if objectType == "TABLE" { + details["INCREASE_INVALID_COUNT"] = false + } + return newIssueInstance(alterTableAddPKOnPartitionIssue, objectType, objectName, SqlStatement, details) +} + +var expressionPartitionIssue = Issue{ + Type: EXPRESSION_PARTITION_WITH_PK_UK, + TypeName: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", + Suggestion: "Remove the Constriant from the table definition", + GH: "https://github.com/yugabyte/yb-voyager/issues/698", + DocsLink: DOCS_LINK_PREFIX + MYSQL_PREFIX + "#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", +} + +func NewExpressionPartitionIssue(objectType string, objectName string, SqlStatement string) IssueInstance { + return newIssueInstance(expressionPartitionIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var multiColumnListPartition = Issue{ + Type: MULTI_COLUMN_LIST_PARTITION, + TypeName: `cannot use "list" partition strategy with more than one column`, + Suggestion: "Make it a single column partition by list or choose other supported Partitioning methods", + GH: "https://github.com/yugabyte/yb-voyager/issues/699", + DocsLink: DOCS_LINK_PREFIX + MYSQL_PREFIX + "#multi-column-partition-by-list-is-not-supported", +} + +func NewMultiColumnListPartition(objectType string, objectName string, SqlStatement string) IssueInstance { + return newIssueInstance(multiColumnListPartition, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var insufficientColumnsInPKForPartition = Issue{ + Type: INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, + TypeName: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", + Suggestion: "Add all Partition columns to Primary Key", + GH: "https://github.com/yugabyte/yb-voyager/issues/578", + DocsLink: DOCS_LINK_PREFIX + ORACLE_PREFIX + "#partition-key-column-not-part-of-primary-key-columns", +} + +func NewInsufficientColumnInPKForPartition(objectType string, objectName string, SqlStatement string, partitionColumnsNotInPK []string) IssueInstance { + issue := insufficientColumnsInPKForPartition + issue.TypeName = fmt.Sprintf("%s - (%s)", issue.TypeName, strings.Join(partitionColumnsNotInPK, ", ")) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var xmlDatatypeIssue = Issue{ + Type: XML_DATATYPE, + TypeName: "Unsupported datatype - xml", + Suggestion: "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", + GH: "https://github.com/yugabyte/yugabyte-db/issues/1043", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#data-ingestion-on-xml-data-type-is-not-supported", +} + +func NewXMLDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) IssueInstance { + issue := xmlDatatypeIssue + issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var xidDatatypeIssue = Issue{ + Type: XID_DATATYPE, + TypeName: "Unsupported datatype - xid", + Suggestion: "Functions for this type e.g. txid_current are not supported in YugabyteDB yet", + GH: "https://github.com/yugabyte/yugabyte-db/issues/15638", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#xid-functions-is-not-supported", +} + +func NewXIDDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) IssueInstance { + issue := xidDatatypeIssue + issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var postgisDatatypeIssue = Issue{ + Type: POSTGIS_DATATYPES, + TypeName: "Unsupported datatype", + GH: "https://github.com/yugabyte/yugabyte-db/issues/11323", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-yugabytedb", +} + +func NewPostGisDatatypeIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { + issue := postgisDatatypeIssue + issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var unsupportedDatatypesIssue = Issue{ + Type: UNSUPPORTED_DATATYPES, + TypeName: "Unsupported datatype", + GH: "https://github.com/yugabyte/yb-voyager/issues/1731", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-yugabytedb", +} + +func NewUnsupportedDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { + issue := unsupportedDatatypesIssue + issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var unsupportedDatatypesForLiveMigrationIssue = Issue{ + Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION, + TypeName: "Unsupported datatype for Live migration", + GH: "https://github.com/yugabyte/yb-voyager/issues/1731", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-voyager-during-live-migration", +} + +func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { + issue := unsupportedDatatypesForLiveMigrationIssue + issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = Issue{ + Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB, + TypeName: "Unsupported datatype for Live migration with fall-forward/fallback", + GH: "https://github.com/yugabyte/yb-voyager/issues/1731", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-voyager-during-live-migration", +} + +func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { + issue := unsupportedDatatypesForLiveMigrationWithFFOrFBIssue + issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var primaryOrUniqueOnUnsupportedIndexTypesIssue = Issue{ + Type: PK_UK_ON_COMPLEX_DATATYPE, + TypeName: "Primary key and Unique constraint on column '%s' not yet supported", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", + Suggestion: "Refer to the docs link for the workaround", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, +} + +func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, SqlStatement string, typeName string, increaseInvalidCnt bool) IssueInstance { + details := map[string]interface{}{} + //for ALTER not increasing count, but for Create increasing TODO: fix + if !increaseInvalidCnt { + details["INCREASE_INVALID_COUNT"] = false + } + issue := primaryOrUniqueOnUnsupportedIndexTypesIssue + issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, details) +} + +var indexOnComplexDatatypesIssue = Issue{ + Type: INDEX_ON_COMPLEX_DATATYPE, + TypeName: "INDEX on column '%s' not yet supported", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", + Suggestion: "Refer to the docs link for the workaround", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#indexes-on-some-complex-data-types-are-not-supported", +} + +func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string) IssueInstance { + issue := indexOnComplexDatatypesIssue + issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var foreignTableIssue = Issue{ + Type: FOREIGN_TABLE, + TypeName: "Foreign tables require manual intervention.", + GH: "https://github.com/yugabyte/yb-voyager/issues/1627", + Suggestion: "SERVER '%s', and USER MAPPING should be created manually on the target to create and use the foreign table", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#foreign-table-in-the-source-database-requires-server-and-user-mapping", +} + +func NewForeignTableIssue(objectType string, objectName string, SqlStatement string, serverName string) IssueInstance { + issue := foreignTableIssue + issue.Suggestion = fmt.Sprintf(issue.Suggestion, serverName) + return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + +var inheritanceIssue = Issue{ + Type: INHERITANCE, + TypeName: "TABLE INHERITANCE not supported in YugabyteDB", + GH: "https://github.com/YugaByte/yugabyte-db/issues/1129", + DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#table-inheritance-is-not-supported", +} + +func NewInheritanceIssue(objectType string, objectName string, sqlStatement string) IssueInstance { + return newIssueInstance(inheritanceIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} var percentTypeSyntax = Issue{ Type: REFERENCED_TYPE_DECLARATION, TypeName: "Referenced type declaration of variables", diff --git a/yb-voyager/src/queryissue/helpers.go b/yb-voyager/src/queryissue/helpers.go index 849cd15b57..fad34e2de0 100644 --- a/yb-voyager/src/queryissue/helpers.go +++ b/yb-voyager/src/queryissue/helpers.go @@ -53,3 +53,43 @@ var unsupportedXmlFunctions = []string{ */ "xmlconcat2", "xmlvalidate", "xml_in", "xml_out", "xml_recv", "xml_send", // System XML I/O } + +var UnsupportedIndexMethods = []string{ + "gist", + "brin", + "spgist", +} + +// Reference for some of the types https://docs.yugabyte.com/stable/api/ysql/datatypes/ (datatypes with type 1) +var UnsupportedIndexDatatypes = []string{ + "citext", + "tsvector", + "tsquery", + "jsonb", + "inet", + "json", + "macaddr", + "macaddr8", + "cidr", + "bit", // for BIT (n) + "varbit", // for BIT varying (n) + "daterange", + "tsrange", + "tstzrange", + "numrange", + "int4range", + "int8range", + "interval", // same for INTERVAL YEAR TO MONTH and INTERVAL DAY TO SECOND + //Below ones are not supported on PG as well with atleast btree access method. Better to have in our list though + //Need to understand if there is other method or way available in PG to have these index key [TODO] + "circle", + "box", + "line", + "lseg", + "point", + "pg_lsn", + "path", + "polygon", + "txid_snapshot", + // array as well but no need to add it in the list as fetching this type is a different way TODO: handle better with specific types +} diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 1f4239f446..2c1871b637 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -18,6 +18,7 @@ package queryissue import ( "fmt" + "slices" "strings" "github.com/samber/lo" @@ -30,12 +31,79 @@ import ( ) type ParserIssueDetector struct { - // TODO: Add fields here - // e.g. store composite types, etc. for future processing. + /* + this will contain the information in this format: + public.table1 -> { + column1: citext | jsonb | inet | tsquery | tsvector | array + ... + } + schema2.table2 -> { + column3: citext | jsonb | inet | tsquery | tsvector | array + ... + } + Here only those columns on tables are stored which have unsupported type for Index in YB + */ + columnsWithUnsupportedIndexDatatypes map[string]map[string]string + /* + list of composite types with fully qualified typename in the exported schema + */ + compositeTypes []string + /* + list of enum types with fully qualified typename in the exported schema + */ + enumTypes []string + + partitionTablesMap map[string]bool + + // key is partitioned table, value is sqlInfo (sqlstmt, fpath) where the ADD PRIMARY KEY statement resides + primaryConsInAlter map[string]*queryparser.AlterTable + + //Boolean to check if there are any Gin indexes + IsGinIndexPresentInSchema bool } func NewParserIssueDetector() *ParserIssueDetector { - return &ParserIssueDetector{} + return &ParserIssueDetector{ + columnsWithUnsupportedIndexDatatypes: make(map[string]map[string]string), + compositeTypes: make([]string, 0), + enumTypes: make([]string, 0), + partitionTablesMap: make(map[string]bool), + primaryConsInAlter: make(map[string]*queryparser.AlterTable), + } +} + +func (p *ParserIssueDetector) GetCompositeTypes() []string { + return p.compositeTypes +} + +func (p *ParserIssueDetector) GetEnumTypes() []string { + return p.enumTypes +} + +func (p *ParserIssueDetector) GetAllIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { + issues, err := p.getAllIssues(query) + if err != nil { + return issues, err + } + + return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) +} + +func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, error) { + plpgsqlIssues, err := p.getPLPGSQLIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting plpgsql issues: %v", err) + } + + dmlIssues, err := p.getDMLIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting dml issues: %v", err) + } + ddlIssues, err := p.getDDLIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting ddl issues: %v", err) + } + return lo.Flatten([][]issue.IssueInstance{plpgsqlIssues, dmlIssues, ddlIssues}), nil } func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []issue.IssueInstance, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { @@ -52,21 +120,21 @@ func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []issue. return filteredIssues, nil } -func (p *ParserIssueDetector) GetAllIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { - issues, err := p.getAllIssues(query) +func (p *ParserIssueDetector) GetAllPLPGSQLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { + issues, err := p.getPLPGSQLIssues(query) if err != nil { - return issues, err + return issues, nil } return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) } -func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]issue.IssueInstance, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) } - + //TODO handle this in DDLPARSER, DDLIssueDetector if queryparser.IsPLPGSQLObject(parseTree) { objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) plpgsqlQueries, err := queryparser.GetAllPLPGSQLStatements(query) @@ -105,7 +173,7 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, if err != nil { return nil, fmt.Errorf("error deparsing a select stmt: %v", err) } - issues, err := p.getAllIssues(selectStmtQuery) + issues, err := p.getDMLIssues(selectStmtQuery) if err != nil { return nil, err } @@ -118,10 +186,109 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, return i }), nil } + return nil, nil +} - issues, err := p.getDMLIssues(query) +func (p *ParserIssueDetector) ParseRequiredDDLs(query string) error { + parseTree, err := queryparser.Parse(query) + if err != nil { + return fmt.Errorf("error parsing a query: %v", err) + } + ddlObj, err := queryparser.ProcessDDL(parseTree) + if err != nil { + return fmt.Errorf("error parsing DDL: %w", err) + } + + switch ddlObj.(type) { + case *queryparser.AlterTable: + alter, _ := ddlObj.(*queryparser.AlterTable) + if alter.ConstraintType == queryparser.PRIMARY_CONSTR_TYPE { + //For the case ALTER and CREATE are not not is expected order where ALTER is before CREATE + alter.Query = query + p.primaryConsInAlter[alter.GetObjectName()] = alter + } + case *queryparser.Table: + table, _ := ddlObj.(*queryparser.Table) + if table.IsPartitioned { + p.partitionTablesMap[table.GetObjectName()] = true + } + + for _, col := range table.Columns { + isUnsupportedType := slices.Contains(UnsupportedIndexDatatypes, col.TypeName) + isUDTType := slices.Contains(p.compositeTypes, col.GetFullTypeName()) + switch true { + case col.IsArrayType: + //For Array types and storing the type as "array" as of now we can enhance the to have specific type e.g. INT4ARRAY + _, ok := p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()] + if !ok { + p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()] = make(map[string]string) + } + p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()][col.ColumnName] = "array" + case isUnsupportedType || isUDTType: + _, ok := p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()] + if !ok { + p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()] = make(map[string]string) + } + p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()][col.ColumnName] = col.TypeName + if isUDTType { //For UDTs + p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()][col.ColumnName] = "user_defined_type" + } + } + } + + case *queryparser.CreateType: + typeObj, _ := ddlObj.(*queryparser.CreateType) + if typeObj.IsEnum { + p.enumTypes = append(p.enumTypes, typeObj.GetObjectName()) + } else { + p.compositeTypes = append(p.compositeTypes, typeObj.GetObjectName()) + } + case *queryparser.Index: + index, _ := ddlObj.(*queryparser.Index) + if index.AccessMethod == GIN_ACCESS_METHOD { + p.IsGinIndexPresentInSchema = true + } + } + return nil +} + +func (p *ParserIssueDetector) GetDDLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { + issues, err := p.getDDLIssues(query) + if err != nil { + return issues, nil + } + + return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) + +} + +func (p *ParserIssueDetector) getDDLIssues(query string) ([]issue.IssueInstance, error) { + parseTree, err := queryparser.Parse(query) if err != nil { - return nil, fmt.Errorf("error getting DML issues: %w", err) + return nil, fmt.Errorf("error parsing a query: %v", err) + } + // Parse the query into a DDL object + ddlObj, err := queryparser.ProcessDDL(parseTree) + if err != nil { + return nil, fmt.Errorf("error parsing DDL: %w", err) + } + // Get the appropriate issue detector + detector, err := p.GetDDLDetector(ddlObj) + if err != nil { + return nil, fmt.Errorf("error getting issue detector: %w", err) + } + + // Detect issues + issues, err := detector.DetectIssues(ddlObj) + if err != nil { + return nil, fmt.Errorf("error detecting issues: %w", err) + } + + // Add the original query to each issue + for i := range issues { + if issues[i].SqlStatement == "" { + issues[i].SqlStatement = query + } } return issues, nil } diff --git a/yb-voyager/src/queryissue/unsupported_ddls.go b/yb-voyager/src/queryissue/unsupported_ddls.go new file mode 100644 index 0000000000..31e9ae531a --- /dev/null +++ b/yb-voyager/src/queryissue/unsupported_ddls.go @@ -0,0 +1,579 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queryissue + +import ( + "fmt" + "slices" + + "github.com/samber/lo" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +// DDLIssueDetector interface defines methods for detecting issues in DDL objects +type DDLIssueDetector interface { + DetectIssues(queryparser.DDLObject) ([]issue.IssueInstance, error) +} + +//=============TABLE ISSUE DETECTOR =========================== + +// TableIssueDetector handles detection of table-related issues +type TableIssueDetector struct { + ParserIssueDetector +} + +func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + table, ok := obj.(*queryparser.Table) + if !ok { + return nil, fmt.Errorf("invalid object type: expected Table") + } + + var issues []issue.IssueInstance + + // Check for generated columns + if len(table.GeneratedColumns) > 0 { + issues = append(issues, issue.NewGeneratedColumnsIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", // query string + table.GeneratedColumns, + )) + } + + // Check for unlogged table + if table.IsUnlogged { + issues = append(issues, issue.NewUnloggedTableIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", // query string + )) + } + + if table.IsInherited { + issues = append(issues, issue.NewInheritanceIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + )) + } + + if len(table.Constraints) > 0 { + + for _, c := range table.Constraints { + if c.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { + issues = append(issues, issue.NewExclusionConstraintIssue( + issue.TABLE_OBJECT_TYPE, + fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + "", + )) + } + + if c.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && c.IsDeferrable { + issues = append(issues, issue.NewDeferrableConstraintIssue( + issue.TABLE_OBJECT_TYPE, + fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + "", + )) + } + + if c.IsPrimaryKeyORUniqueConstraint() { + for _, col := range c.Columns { + unsupportedColumnsForTable, ok := d.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()] + if !ok { + break + } + + typeName, ok := unsupportedColumnsForTable[col] + if !ok { + continue + } + issues = append(issues, issue.NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( + issue.TABLE_OBJECT_TYPE, + fmt.Sprintf("%s, constraint: %s", table.GetObjectName(), c.ConstraintName), + "", + typeName, + true, + )) + } + } + } + } + for _, col := range table.Columns { + liveUnsupportedDatatypes := srcdb.GetPGLiveMigrationUnsupportedDatatypes() + liveWithFfOrFbUnsupportedDatatypes := srcdb.GetPGLiveMigrationWithFFOrFBUnsupportedDatatypes() + + isUnsupportedDatatype := utils.ContainsAnyStringFromSlice(srcdb.PostgresUnsupportedDataTypes, col.TypeName) + isUnsupportedDatatypeInLive := utils.ContainsAnyStringFromSlice(liveUnsupportedDatatypes, col.TypeName) + + isUnsupportedDatatypeInLiveWithFFOrFBList := utils.ContainsAnyStringFromSlice(liveWithFfOrFbUnsupportedDatatypes, col.TypeName) + isUDTDatatype := utils.ContainsAnyStringFromSlice(d.compositeTypes, col.GetFullTypeName()) //if type is array + isEnumDatatype := utils.ContainsAnyStringFromSlice(d.enumTypes, col.GetFullTypeName()) //is ENUM type + isArrayOfEnumsDatatype := col.IsArrayType && isEnumDatatype + isUnsupportedDatatypeInLiveWithFFOrFB := isUnsupportedDatatypeInLiveWithFFOrFBList || isUDTDatatype || isArrayOfEnumsDatatype + + if isUnsupportedDatatype { + reportUnsupportedDatatypes(col, issue.TABLE_OBJECT_TYPE, table.GetObjectName(), &issues) + } else if isUnsupportedDatatypeInLive { + issues = append(issues, issue.NewUnsupportedDatatypesForLMIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + col.TypeName, + col.ColumnName, + )) + } else if isUnsupportedDatatypeInLiveWithFFOrFB { + //reporting only for TABLE Type as we don't deal with FOREIGN TABLE in live migration + reportTypeName := col.GetFullTypeName() + if col.IsArrayType { // For Array cases to make it clear in issue + reportTypeName = fmt.Sprintf("%s[]", reportTypeName) + } + issues = append(issues, issue.NewUnsupportedDatatypesForLMWithFFOrFBIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + reportTypeName, + col.ColumnName, + )) + } + } + + if table.IsPartitioned { + + /* + 1. Adding PK to Partitioned Table (in cases where ALTER is before create) + 2. Expression partitions are not allowed if PK/UNIQUE columns are there is table + 3. List partition strategy is not allowed with multi-column partitions. + 4. Partition columns should all be included in Primary key set if any on table. + */ + alterAddPk := d.primaryConsInAlter[table.GetObjectName()] + if alterAddPk != nil { + issues = append(issues, issue.NewAlterTableAddPKOnPartiionIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + alterAddPk.Query, + )) + } + primaryKeyColumns := table.PrimaryKeyColumns() + uniqueKeyColumns := table.UniqueKeyColumns() + + if table.IsExpressionPartition && (len(primaryKeyColumns) > 0 || len(uniqueKeyColumns) > 0) { + issues = append(issues, issue.NewExpressionPartitionIssue( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + )) + } + + if table.PartitionStrategy == queryparser.LIST_PARTITION && + len(table.PartitionColumns) > 1 { + issues = append(issues, issue.NewMultiColumnListPartition( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + )) + } + partitionColumnsNotInPK, _ := lo.Difference(table.PartitionColumns, primaryKeyColumns) + if len(primaryKeyColumns) > 0 && len(partitionColumnsNotInPK) > 0 { + issues = append(issues, issue.NewInsufficientColumnInPKForPartition( + issue.TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + partitionColumnsNotInPK, + )) + } + } + + return issues, nil +} + +func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, objName string, issues *[]issue.IssueInstance) { + switch col.TypeName { + case "xml": + *issues = append(*issues, issue.NewXMLDatatypeIssue( + objType, + objName, + "", + col.ColumnName, + )) + case "xid": + *issues = append(*issues, issue.NewXIDDatatypeIssue( + objType, + objName, + "", + col.ColumnName, + )) + case "geometry", "geography", "box2d", "box3d", "topogeometry": + *issues = append(*issues, issue.NewPostGisDatatypeIssue( + objType, + objName, + "", + col.TypeName, + col.ColumnName, + )) + default: + *issues = append(*issues, issue.NewUnsupportedDatatypesIssue( + objType, + objName, + "", + col.TypeName, + col.ColumnName, + )) + } +} + +//=============FOREIGN TABLE ISSUE DETECTOR =========================== + +//ForeignTableIssueDetector handles detection Foreign table issues + +type ForeignTableIssueDetector struct{} + +func (f *ForeignTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + foreignTable, ok := obj.(*queryparser.ForeignTable) + if !ok { + return nil, fmt.Errorf("invalid object type: expected Foreign Table") + } + issues := make([]issue.IssueInstance, 0) + + issues = append(issues, issue.NewForeignTableIssue( + issue.FOREIGN_TABLE_OBJECT_TYPE, + foreignTable.GetObjectName(), + "", + foreignTable.ServerName, + )) + + for _, col := range foreignTable.Columns { + isUnsupportedDatatype := utils.ContainsAnyStringFromSlice(srcdb.PostgresUnsupportedDataTypes, col.TypeName) + if isUnsupportedDatatype { + reportUnsupportedDatatypes(col, issue.FOREIGN_TABLE_OBJECT_TYPE, foreignTable.GetObjectName(), &issues) + } + } + + return issues, nil + +} + +//=============INDEX ISSUE DETECTOR =========================== + +// IndexIssueDetector handles detection of index-related issues +type IndexIssueDetector struct { + ParserIssueDetector +} + +func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + index, ok := obj.(*queryparser.Index) + if !ok { + return nil, fmt.Errorf("invalid object type: expected Index") + } + + var issues []issue.IssueInstance + + // Check for unsupported index methods + if slices.Contains(UnsupportedIndexMethods, index.AccessMethod) { + issues = append(issues, issue.NewUnsupportedIndexMethodIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", // query string + index.AccessMethod, + )) + } + + // Check for storage parameters + if index.NumStorageOptions > 0 { + issues = append(issues, issue.NewStorageParameterIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", // query string + )) + } + + //GinVariations + if index.AccessMethod == GIN_ACCESS_METHOD { + if len(index.Params) > 1 { + issues = append(issues, issue.NewMultiColumnGinIndexIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", + )) + } else { + //In case only one Param is there + param := index.Params[0] + if param.SortByOrder != queryparser.DEFAULT_SORTING_ORDER { + issues = append(issues, issue.NewOrderedGinIndexIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", + )) + } + } + } + + //Index on complex datatypes + /* + cases covered + 1. normal index on column with these types + 2. expression index with casting of unsupported column to supported types [No handling as such just to test as colName will not be there] + 3. expression index with casting to unsupported types + 4. normal index on column with UDTs + 5. these type of indexes on different access method like gin etc.. [TODO to explore more, for now not reporting the indexes on anyother access method than btree] + */ + _, ok = d.columnsWithUnsupportedIndexDatatypes[index.GetTableName()] + if ok && index.AccessMethod == BTREE_ACCESS_METHOD { // Right now not reporting any other access method issues with such types. + for _, param := range index.Params { + if param.IsExpression { + isUnsupportedType := slices.Contains(UnsupportedIndexDatatypes, param.ExprCastTypeName) + isUDTType := slices.Contains(d.compositeTypes, param.GetFullExprCastTypeName()) + if param.IsExprCastArrayType { + issues = append(issues, issue.NewIndexOnComplexDatatypesIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", + "array", + )) + } else if isUnsupportedType || isUDTType { + reportTypeName := param.ExprCastTypeName + if isUDTType { + reportTypeName = "user_defined_type" + } + issues = append(issues, issue.NewIndexOnComplexDatatypesIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", + reportTypeName, + )) + } + + } else { + colName := param.ColName + typeName, ok := d.columnsWithUnsupportedIndexDatatypes[index.GetTableName()][colName] + if !ok { + continue + } + issues = append(issues, issue.NewIndexOnComplexDatatypesIssue( + issue.INDEX_OBJECT_TYPE, + index.GetObjectName(), + "", + typeName, + )) + } + } + } + + return issues, nil +} + +//=============ALTER TABLE ISSUE DETECTOR =========================== + +// AlterTableIssueDetector handles detection of alter table-related issues +type AlterTableIssueDetector struct { + ParserIssueDetector +} + +func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + alter, ok := obj.(*queryparser.AlterTable) + if !ok { + return nil, fmt.Errorf("invalid object type: expected AlterTable") + } + + var issues []issue.IssueInstance + + switch alter.AlterType { + case queryparser.SET_OPTIONS: + if alter.NumSetAttributes > 0 { + issues = append(issues, issue.NewSetAttributeIssue( + issue.TABLE_OBJECT_TYPE, + alter.GetObjectName(), + "", // query string + )) + } + case queryparser.ADD_CONSTRAINT: + if alter.NumStorageOptions > 0 { + issues = append(issues, issue.NewStorageParameterIssue( + issue.TABLE_OBJECT_TYPE, + alter.GetObjectName(), + "", // query string + )) + } + if alter.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { + issues = append(issues, issue.NewExclusionConstraintIssue( + issue.TABLE_OBJECT_TYPE, + fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + "", + )) + } + if alter.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && alter.IsDeferrable { + issues = append(issues, issue.NewDeferrableConstraintIssue( + issue.TABLE_OBJECT_TYPE, + fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + "", + )) + } + + if alter.ConstraintType == queryparser.PRIMARY_CONSTR_TYPE && + aid.partitionTablesMap[alter.GetObjectName()] { + issues = append(issues, issue.NewAlterTableAddPKOnPartiionIssue( + issue.TABLE_OBJECT_TYPE, + alter.GetObjectName(), + "", + )) + } + + if alter.AddPrimaryKeyOrUniqueCons() { + for _, col := range alter.ConstraintColumns { + unsupportedColumnsForTable, ok := aid.columnsWithUnsupportedIndexDatatypes[alter.GetObjectName()] + if !ok { + break + } + + typeName, ok := unsupportedColumnsForTable[col] + if !ok { + continue + } + issues = append(issues, issue.NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( + issue.TABLE_OBJECT_TYPE, + fmt.Sprintf("%s, constraint: %s", alter.GetObjectName(), alter.ConstraintName), + "", + typeName, + false, + )) + } + + } + case queryparser.DISABLE_RULE: + issues = append(issues, issue.NewDisableRuleIssue( + issue.TABLE_OBJECT_TYPE, + alter.GetObjectName(), + "", // query string + alter.RuleName, + )) + case queryparser.CLUSTER_ON: + issues = append(issues, issue.NewClusterONIssue( + issue.TABLE_OBJECT_TYPE, + alter.GetObjectName(), + "", // query string + )) + } + + return issues, nil +} + +//=============POLICY ISSUE DETECTOR =========================== + +// PolicyIssueDetector handles detection of Create policy issues +type PolicyIssueDetector struct{} + +func (p *PolicyIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + policy, ok := obj.(*queryparser.Policy) + if !ok { + return nil, fmt.Errorf("invalid object type: expected Policy") + } + issues := make([]issue.IssueInstance, 0) + if len(policy.RoleNames) > 0 { + issues = append(issues, issue.NewPolicyRoleIssue( + issue.POLICY_OBJECT_TYPE, + policy.GetObjectName(), + "", + policy.RoleNames, + )) + } + return issues, nil +} + +//=============TRIGGER ISSUE DETECTOR =========================== + +// TriggerIssueDetector handles detection of Create Trigger issues +type TriggerIssueDetector struct { + ParserIssueDetector +} + +func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + trigger, ok := obj.(*queryparser.Trigger) + if !ok { + return nil, fmt.Errorf("invalid object type: expected Trigger") + } + issues := make([]issue.IssueInstance, 0) + + if trigger.IsConstraint { + issues = append(issues, issue.NewConstraintTriggerIssue( + issue.TRIGGER_OBJECT_TYPE, + trigger.GetObjectName(), + "", + )) + } + + if trigger.NumTransitionRelations > 0 { + issues = append(issues, issue.NewReferencingClauseTrigIssue( + issue.TRIGGER_OBJECT_TYPE, + trigger.GetObjectName(), + "", + )) + } + + if trigger.IsBeforeRowTrigger() && tid.partitionTablesMap[trigger.GetTableName()] { + issues = append(issues, issue.NewBeforeRowOnPartitionTableIssue( + issue.TRIGGER_OBJECT_TYPE, + trigger.GetObjectName(), + "", + )) + } + + return issues, nil +} + +//=============NO-OP ISSUE DETECTOR =========================== + +// Need to handle all the cases for which we don't have any issues detector +type NoOpIssueDetector struct{} + +func (n *NoOpIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + return nil, nil +} + +func (p *ParserIssueDetector) GetDDLDetector(obj queryparser.DDLObject) (DDLIssueDetector, error) { + switch obj.(type) { + case *queryparser.Table: + return &TableIssueDetector{ + ParserIssueDetector: *p, + }, nil + case *queryparser.Index: + return &IndexIssueDetector{ + ParserIssueDetector: *p, + }, nil + case *queryparser.AlterTable: + return &AlterTableIssueDetector{ + ParserIssueDetector: *p, + }, nil + case *queryparser.Policy: + return &PolicyIssueDetector{}, nil + case *queryparser.Trigger: + return &TriggerIssueDetector{ + ParserIssueDetector: *p, + }, nil + case *queryparser.ForeignTable: + return &ForeignTableIssueDetector{}, nil + default: + return &NoOpIssueDetector{}, nil + } +} + +const ( + GIN_ACCESS_METHOD = "gin" + BTREE_ACCESS_METHOD = "btree" +) diff --git a/yb-voyager/src/queryparser/ddl_processor.go b/yb-voyager/src/queryparser/ddl_processor.go new file mode 100644 index 0000000000..fb0ebf0f95 --- /dev/null +++ b/yb-voyager/src/queryparser/ddl_processor.go @@ -0,0 +1,880 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryparser + +import ( + "fmt" + "slices" + "strings" + + pg_query "github.com/pganalyze/pg_query_go/v5" + "github.com/samber/lo" +) + +// Base parser interface +/* +Whenever adding a new DDL type to prasing for detecting issues, need to extend this DDLProcessor +with the Process() function to adding logic to get the required information out from the parseTree of that DDL +and store it in a DDLObject struct +*/ +type DDLProcessor interface { + Process(*pg_query.ParseResult) (DDLObject, error) +} + +// Base DDL object interface +/* +Whenever adding a new DDL type, You need to extend this DDLObject struct to be extended for that object type +with the required for storing the information which should have these required function also extended for the objeect Name and schema name +*/ +type DDLObject interface { + GetObjectName() string + GetSchemaName() string +} + +//=========== TABLE PROCESSOR ================================ + +// TableProcessor handles parsing CREATE TABLE statements +type TableProcessor struct{} + +func NewTableProcessor() *TableProcessor { + return &TableProcessor{} +} + +/* +e.g. CREATE TABLE "Test"( + + id int, + room_id int, + time_range tsrange, + room_id1 int, + time_range1 tsrange + EXCLUDE USING gist (room_id WITH =, time_range WITH &&), + EXCLUDE USING gist (room_id1 WITH =, time_range1 WITH &&) + ); + +create_stmt:{relation:{relname:"Test" inh:true relpersistence:"p" location:14} table_elts:...table_elts:{constraint:{contype:CONSTR_EXCLUSION +location:226 exclusions:{list:{items:{index_elem:{name:"room_id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} +items:{list:{items:{string:{sval:"="}}}}}} exclusions:{list:{items:{index_elem:{name:"time_range" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} +items:{list:{items:{string:{sval:"&&"}}}}}} access_method:"gist"}} table_elts:{constraint:{contype:CONSTR_EXCLUSION location:282 exclusions:{list: +{items:{index_elem:{name:"room_id1" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} items:{list:{items:{string:{sval:"="}}}}}} +exclusions:{list:{items:{index_elem:{name:"time_range1" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} items:{list:{items:{string:{sval:"&&"}}}}}} +access_method:"gist"}} oncommit:ONCOMMIT_NOOP}} stmt_len:365} + +here we are iterating over all the table_elts - table elements and which are comma separated column info in +the DDL so each column has column_def(column definition) in the parse tree but in case it is a constraint, the column_def +is nil. + +e.g. In case if PRIMARY KEY is included in column definition + + CREATE TABLE example2 ( + id numeric NOT NULL PRIMARY KEY, + country_code varchar(3), + record_type varchar(5) + +) PARTITION BY RANGE (country_code, record_type) ; +stmts:{stmt:{create_stmt:{relation:{relname:"example2" inh:true relpersistence:"p" location:193} table_elts:{column_def:{colname:"id" +type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"numeric"}} typemod:-1 location:208} is_local:true +constraints:{constraint:{contype:CONSTR_NOTNULL location:216}} constraints:{constraint:{contype:CONSTR_PRIMARY location:225}} +location:205}} ... partspec:{strategy:PARTITION_STRATEGY_RANGE +part_params:{partition_elem:{name:"country_code" location:310}} part_params:{partition_elem:{name:"record_type" location:324}} +location:290} oncommit:ONCOMMIT_NOOP}} stmt_location:178 stmt_len:159} + +In case if PRIMARY KEY in column list CREATE TABLE example1 (..., PRIMARY KEY(id,country_code) ) PARTITION BY RANGE (country_code, record_type); +stmts:{stmt:{create_stmt:{relation:{relname:"example1" inh:true relpersistence:"p" location:15} table_elts:{column_def:{colname:"id" +type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"numeric"}} ... table_elts:{constraint:{contype:CONSTR_PRIMARY +location:98 keys:{string:{sval:"id"}} keys:{string:{sval:"country_code"}}}} partspec:{strategy:PARTITION_STRATEGY_RANGE +part_params:{partition_elem:{name:"country_code" location:150}} part_params:{partition_elem:{name:"record_type" ... +*/ +func (tableProcessor *TableProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + createTableNode, ok := getCreateTableStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE TABLE statement") + } + + table := &Table{ + SchemaName: createTableNode.CreateStmt.Relation.Schemaname, + TableName: createTableNode.CreateStmt.Relation.Relname, + /* + e.g CREATE UNLOGGED TABLE tbl_unlogged (id int, val text); + stmt:{create_stmt:{relation:{schemaname:"public" relname:"tbl_unlogged" inh:true relpersistence:"u" location:19} + */ + IsUnlogged: createTableNode.CreateStmt.Relation.GetRelpersistence() == "u", + IsPartitioned: createTableNode.CreateStmt.GetPartspec() != nil, + IsInherited: tableProcessor.checkInheritance(createTableNode), + GeneratedColumns: make([]string, 0), + Constraints: make([]TableConstraint, 0), + PartitionColumns: make([]string, 0), + } + + // Parse columns and their properties + for _, element := range createTableNode.CreateStmt.TableElts { + if element.GetColumnDef() != nil { + if tableProcessor.isGeneratedColumn(element.GetColumnDef()) { + table.GeneratedColumns = append(table.GeneratedColumns, element.GetColumnDef().Colname) + } + colName := element.GetColumnDef().GetColname() + + typeNames := element.GetColumnDef().GetTypeName().GetNames() + typeName, typeSchemaName := getTypeNameAndSchema(typeNames) + /* + e.g. CREATE TABLE test_xml_type(id int, data xml); + relation:{relname:"test_xml_type" inh:true relpersistence:"p" location:15} table_elts:{column_def:{colname:"id" + type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:32} + is_local:true location:29}} table_elts:{column_def:{colname:"data" type_name:{names:{string:{sval:"xml"}} + typemod:-1 location:42} is_local:true location:37}} oncommit:ONCOMMIT_NOOP}} + + here checking the type of each column as type definition can be a list names for types which are native e.g. int + it has type names - [pg_catalog, int4] both to determine but for complex types like text,json or xml etc. if doesn't have + info about pg_catalog. so checking the 0th only in case XML/XID to determine the type and report + */ + table.Columns = append(table.Columns, TableColumn{ + ColumnName: colName, + TypeName: typeName, + TypeSchema: typeSchemaName, + IsArrayType: isArrayType(element.GetColumnDef().GetTypeName()), + }) + + constraints := element.GetColumnDef().GetConstraints() + if constraints != nil { + for idx, c := range constraints { + constraint := c.GetConstraint() + if slices.Contains(deferrableConstraintsList, constraint.Contype) { + /* + e.g. create table unique_def_test(id int UNIQUE DEFERRABLE, c1 int); + + create_stmt:{relation:{relname:"unique_def_test" inh:true relpersistence:"p" location:15} + table_elts:{column_def:{colname:"id" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} + typemod:-1 location:34} is_local:true constraints:{constraint:{contype:CONSTR_UNIQUE location:38}} + constraints:{constraint:{contype:CONSTR_ATTR_DEFERRABLE location:45}} location:31}} .... + + here checking the case where this clause is in column definition so iterating over each column_def and in that + constraint type has deferrable or not and also it should not be a foreign constraint as Deferrable on FKs are + supported. + */ + if idx > 0 { + lastConstraint := table.Constraints[len(table.Constraints)-1] + lastConstraint.IsDeferrable = true + table.Constraints[len(table.Constraints)-1] = lastConstraint + } + } else { + table.addConstraint(constraint.Contype, []string{colName}, constraint.Conname, false) + } + } + } + + } else if element.GetConstraint() != nil { + /* + e.g. create table uniquen_def_test1(id int, c1 int, UNIQUE(id) DEFERRABLE INITIALLY DEFERRED); + {create_stmt:{relation:{relname:"unique_def_test1" inh:true relpersistence:"p" location:80} table_elts:{column_def:{colname:"id" + type_name:{.... names:{string:{sval:"int4"}} typemod:-1 location:108} is_local:true location:105}} + table_elts:{constraint:{contype:CONSTR_UNIQUE deferrable:true initdeferred:true location:113 keys:{string:{sval:"id"}}}} .. + + here checking the case where this constraint is at the at the end as a constraint only, so checking deferrable field in constraint + in case of its not a FK. + */ + constraint := element.GetConstraint() + conType := element.GetConstraint().Contype + columns := parseColumnsFromKeys(constraint.GetKeys()) + if conType == EXCLUSION_CONSTR_TYPE { + //In case CREATE DDL has EXCLUDE USING gist(room_id '=', time_range WITH &&) - it will be included in columns but won't have columnDef as its a constraint + exclusions := constraint.GetExclusions() + //exclusions:{list:{items:{index_elem:{name:"room_id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} + //items:{list:{items:{string:{sval:"="}}}}}} + columns = tableProcessor.parseColumnsFromExclusions(exclusions) + } + table.addConstraint(conType, columns, constraint.Conname, constraint.Deferrable) + + } + } + + if table.IsPartitioned { + + partitionElements := createTableNode.CreateStmt.GetPartspec().GetPartParams() + table.PartitionStrategy = createTableNode.CreateStmt.GetPartspec().GetStrategy() + + for _, partElem := range partitionElements { + if partElem.GetPartitionElem().GetExpr() != nil { + table.IsExpressionPartition = true + } else { + table.PartitionColumns = append(table.PartitionColumns, partElem.GetPartitionElem().GetName()) + } + } + } + + return table, nil +} + +func (tableProcessor *TableProcessor) checkInheritance(createTableNode *pg_query.Node_CreateStmt) bool { + /* + CREATE TABLE Test(id int, name text) inherits(test_parent); + stmts:{stmt:{create_stmt:{relation:{relname:"test" inh:true relpersistence:"p" location:13} table_elts:{column_def:{colname:"id" .... + inh_relations:{range_var:{relname:"test_parent" inh:true relpersistence:"p" location:46}} oncommit:ONCOMMIT_NOOP}} stmt_len:58} + + CREATE TABLE accounts_list_partitioned_p_northwest PARTITION OF accounts_list_partitioned FOR VALUES IN ('OR', 'WA'); + version:160001 stmts:{stmt:{create_stmt:{relation:{relname:"accounts_list_partitioned_p_northwest" inh:true relpersistence:"p" location:14} + inh_relations:{range_var:{relname:"accounts_list_partitioned" inh:true relpersistence:"p" location:65}} partbound:{strategy:"l" listdatums:{a_const:{sval:{sval:"OR"} location:106}} + listdatums:{a_const:{sval:{sval:"WA"} location:112}} location:102} oncommit:ONCOMMIT_NOOP}} + */ + inheritsRel := createTableNode.CreateStmt.GetInhRelations() + if inheritsRel != nil { + isPartitionOf := createTableNode.CreateStmt.GetPartbound() != nil + return !isPartitionOf + } + return false +} + +func (tableProcessor *TableProcessor) parseColumnsFromExclusions(list []*pg_query.Node) []string { + var res []string + for _, k := range list { + res = append(res, k.GetList().GetItems()[0].GetIndexElem().Name) // every first element of items in exclusions will be col name + } + return res +} + +func parseColumnsFromKeys(keys []*pg_query.Node) []string { + var res []string + for _, k := range keys { + res = append(res, k.GetString_().Sval) + } + return res + +} + +func (tableProcessor *TableProcessor) isGeneratedColumn(colDef *pg_query.ColumnDef) bool { + for _, constraint := range colDef.Constraints { + if constraint.GetConstraint().Contype == pg_query.ConstrType_CONSTR_GENERATED { + return true + } + } + return false +} + +type Table struct { + SchemaName string + TableName string + IsUnlogged bool + IsInherited bool + IsPartitioned bool + Columns []TableColumn + IsExpressionPartition bool + PartitionStrategy pg_query.PartitionStrategy + PartitionColumns []string + GeneratedColumns []string + Constraints []TableConstraint +} + +type TableColumn struct { + ColumnName string + TypeName string + TypeSchema string + IsArrayType bool +} + +func (tc *TableColumn) GetFullTypeName() string { + return lo.Ternary(tc.TypeSchema != "", tc.TypeSchema+"."+tc.TypeName, tc.TypeName) +} + +type TableConstraint struct { + ConstraintType pg_query.ConstrType + ConstraintName string + IsDeferrable bool + Columns []string +} + +func (c *TableConstraint) IsPrimaryKeyORUniqueConstraint() bool { + return c.ConstraintType == PRIMARY_CONSTR_TYPE || c.ConstraintType == UNIQUE_CONSTR_TYPE +} + +func (c *TableConstraint) generateConstraintName(tableName string) string { + suffix := "" + //Deferrable is only applicable to following constraint + //https://www.postgresql.org/docs/current/sql-createtable.html#:~:text=Currently%2C%20only%20UNIQUE%2C%20PRIMARY%20KEY%2C%20EXCLUDE%2C%20and%20REFERENCES + switch c.ConstraintType { + case pg_query.ConstrType_CONSTR_UNIQUE: + suffix = "_key" + case pg_query.ConstrType_CONSTR_PRIMARY: + suffix = "_pkey" + case pg_query.ConstrType_CONSTR_EXCLUSION: + suffix = "_excl" + case pg_query.ConstrType_CONSTR_FOREIGN: + suffix = "_fkey" + } + + return fmt.Sprintf("%s_%s%s", tableName, strings.Join(c.Columns, "_"), suffix) +} + +func (t *Table) GetObjectName() string { + qualifiedTable := lo.Ternary(t.SchemaName != "", fmt.Sprintf("%s.%s", t.SchemaName, t.TableName), t.TableName) + return qualifiedTable +} +func (t *Table) GetSchemaName() string { return t.SchemaName } + +func (t *Table) PrimaryKeyColumns() []string { + for _, c := range t.Constraints { + if c.ConstraintType == PRIMARY_CONSTR_TYPE { + return c.Columns + } + } + return []string{} +} + +func (t *Table) UniqueKeyColumns() []string { + uniqueCols := make([]string, 0) + for _, c := range t.Constraints { + if c.ConstraintType == UNIQUE_CONSTR_TYPE { + uniqueCols = append(uniqueCols, c.Columns...) + } + } + return uniqueCols +} + +func (t *Table) addConstraint(conType pg_query.ConstrType, columns []string, specifiedConName string, deferrable bool) { + tc := TableConstraint{ + ConstraintType: conType, + Columns: columns, + IsDeferrable: deferrable, + } + generatedConName := tc.generateConstraintName(t.GetObjectName()) + conName := lo.Ternary(specifiedConName == "", generatedConName, specifiedConName) + tc.ConstraintName = conName + t.Constraints = append(t.Constraints, tc) +} + +//===========FOREIGN TABLE PROCESSOR ================================ + +type ForeignTableProcessor struct{} + +func NewForeignTableProcessor() *ForeignTableProcessor { + return &ForeignTableProcessor{} +} + +func (ftProcessor *ForeignTableProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + foreignTableNode, ok := getForeignTableStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE FOREIGN TABLE statement") + } + baseStmt := foreignTableNode.CreateForeignTableStmt.BaseStmt + relation := baseStmt.Relation + table := Table{ + TableName: relation.GetRelname(), + SchemaName: relation.GetSchemaname(), + //Not populating rest info + } + for _, element := range baseStmt.TableElts { + if element.GetColumnDef() != nil { + colName := element.GetColumnDef().GetColname() + + typeNames := element.GetColumnDef().GetTypeName().GetNames() + typeName, typeSchemaName := getTypeNameAndSchema(typeNames) + table.Columns = append(table.Columns, TableColumn{ + ColumnName: colName, + TypeName: typeName, + TypeSchema: typeSchemaName, + IsArrayType: isArrayType(element.GetColumnDef().GetTypeName()), + }) + } + } + return &ForeignTable{ + Table: table, + ServerName: foreignTableNode.CreateForeignTableStmt.GetServername(), + }, nil + +} + +type ForeignTable struct { + Table + ServerName string +} + +func (f *ForeignTable) GetObjectName() string { + return lo.Ternary(f.SchemaName != "", f.SchemaName+"."+f.TableName, f.TableName) +} +func (f *ForeignTable) GetSchemaName() string { return f.SchemaName } + +//===========INDEX PROCESSOR ================================ + +// IndexProcessor handles parsing CREATE INDEX statements +type IndexProcessor struct{} + +func NewIndexProcessor() *IndexProcessor { + return &IndexProcessor{} +} + +func (indexProcessor *IndexProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + indexNode, ok := getCreateIndexStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE INDEX statement") + } + + index := &Index{ + SchemaName: indexNode.IndexStmt.Relation.Schemaname, + IndexName: indexNode.IndexStmt.Idxname, + TableName: indexNode.IndexStmt.Relation.Relname, + AccessMethod: indexNode.IndexStmt.AccessMethod, + /* + e.g. CREATE INDEX idx on table_name(id) with (fillfactor='70'); + index_stmt:{idxname:"idx" relation:{relname:"table_name" inh:true relpersistence:"p" location:21} access_method:"btree" + index_params:{index_elem:{name:"id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} + options:{def_elem:{defname:"fillfactor" arg:{string:{sval:"70"}} ... + here again similar to ALTER table Storage parameters options is the high level field in for WITH options. + */ + NumStorageOptions: len(indexNode.IndexStmt.GetOptions()), + Params: indexProcessor.parseIndexParams(indexNode.IndexStmt.IndexParams), + } + + return index, nil +} + +func (indexProcessor *IndexProcessor) parseIndexParams(params []*pg_query.Node) []IndexParam { + /* + e.g. + 1. CREATE INDEX tsvector_idx ON public.documents (title_tsvector, id); + stmt:{index_stmt:{idxname:"tsvector_idx" relation:{schemaname:"public" relname:"documents" inh:true relpersistence:"p" location:510} access_method:"btree" + index_params:{index_elem:{name:"title_tsvector" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} index_params:{index_elem:{name:"id" + ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}}}} stmt_location:479 stmt_len:69 + + 2. CREATE INDEX idx_json ON public.test_json ((data::jsonb)); + stmt:{index_stmt:{idxname:"idx_json" relation:{schemaname:"public" relname:"test_json" inh:true relpersistence:"p" location:703} access_method:"btree" + index_params:{index_elem:{expr:{type_cast:{arg:{column_ref:{fields:{string:{sval:"data"}} location:722}} type_name:{names:{string:{sval:"jsonb"}} typemod:-1 + location:728} location:726}} ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}}}} stmt_location:676 stmt_len:59 + */ + var indexParams []IndexParam + for _, i := range params { + ip := IndexParam{ + SortByOrder: i.GetIndexElem().Ordering, + ColName: i.GetIndexElem().GetName(), + IsExpression: i.GetIndexElem().GetExpr() != nil, + } + if ip.IsExpression { + //For the expression index case to report in case casting to unsupported types #3 + typeNames := i.GetIndexElem().GetExpr().GetTypeCast().GetTypeName().GetNames() + ip.ExprCastTypeName, ip.ExprCastTypeSchema = getTypeNameAndSchema(typeNames) + ip.IsExprCastArrayType = isArrayType(i.GetIndexElem().GetExpr().GetTypeCast().GetTypeName()) + } + indexParams = append(indexParams, ip) + } + return indexParams +} + +type Index struct { + SchemaName string + IndexName string + TableName string + AccessMethod string + NumStorageOptions int + Params []IndexParam +} + +type IndexParam struct { + SortByOrder pg_query.SortByDir + ColName string + IsExpression bool + ExprCastTypeName string //In case of expression and casting to a type + ExprCastTypeSchema string //In case of expression and casting to a type + IsExprCastArrayType bool + //Add more fields +} + +func (indexParam *IndexParam) GetFullExprCastTypeName() string { + return lo.Ternary(indexParam.ExprCastTypeSchema != "", indexParam.ExprCastTypeSchema+"."+indexParam.ExprCastTypeName, indexParam.ExprCastTypeName) +} + +func (i *Index) GetObjectName() string { + return fmt.Sprintf("%s ON %s", i.IndexName, i.GetTableName()) +} +func (i *Index) GetSchemaName() string { return i.SchemaName } + +func (i *Index) GetTableName() string { + return lo.Ternary(i.SchemaName != "", fmt.Sprintf("%s.%s", i.SchemaName, i.TableName), i.TableName) +} + +//===========ALTER TABLE PROCESSOR ================================ + +// AlterTableProcessor handles parsing ALTER TABLE statements +type AlterTableProcessor struct{} + +func NewAlterTableProcessor() *AlterTableProcessor { + return &AlterTableProcessor{} +} + +func (atProcessor *AlterTableProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + alterNode, ok := getAlterStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not an ALTER TABLE statement") + } + + alter := &AlterTable{ + SchemaName: alterNode.AlterTableStmt.Relation.Schemaname, + TableName: alterNode.AlterTableStmt.Relation.Relname, + AlterType: alterNode.AlterTableStmt.Cmds[0].GetAlterTableCmd().GetSubtype(), + } + + // Parse specific alter command + cmd := alterNode.AlterTableStmt.Cmds[0].GetAlterTableCmd() + switch alter.AlterType { + case pg_query.AlterTableType_AT_SetOptions: + /* + e.g. alter table test_1 alter column col1 set (attribute_option=value); + cmds:{alter_table_cmd:{subtype:AT_SetOptions name:"col1" def:{list:{items:{def_elem:{defname:"attribute_option" + arg:{type_name:{names:{string:{sval:"value"}} typemod:-1 location:263}} defaction:DEFELEM_UNSPEC location:246}}}}... + + for set attribute issue we will the type of alter setting the options and in the 'def' definition field which has the + information of the type, we will check if there is any list which will only present in case there is syntax like (...) + */ + alter.NumSetAttributes = len(cmd.GetDef().GetList().GetItems()) + case pg_query.AlterTableType_AT_AddConstraint: + alter.NumStorageOptions = len(cmd.GetDef().GetConstraint().GetOptions()) + /* + e.g. + ALTER TABLE example2 + ADD CONSTRAINT example2_pkey PRIMARY KEY (id); + tmts:{stmt:{alter_table_stmt:{relation:{relname:"example2" inh:true relpersistence:"p" location:693} + cmds:{alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_PRIMARY conname:"example2_pkey" + location:710 keys:{string:{sval:"id"}}}} behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} stmt_location:679 stmt_len:72} + + e.g. ALTER TABLE ONLY public.meeting ADD CONSTRAINT no_time_overlap EXCLUDE USING gist (room_id WITH =, time_range WITH &&); + cmds:{alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_EXCLUSION conname:"no_time_overlap" location:41 + here again same checking the definition of the alter stmt if it has constraint and checking its type + + e.g. ALTER TABLE ONLY public.users ADD CONSTRAINT users_email_key UNIQUE (email) DEFERRABLE; + alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_UNIQUE conname:"users_email_key" + deferrable:true location:196 keys:{string:{sval:"email"}}}} behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} + + similar to CREATE table 2nd case where constraint is at the end of column definitions mentioning the constraint only + so here as well while adding constraint checking the type of constraint and the deferrable field of it. + */ + constraint := cmd.GetDef().GetConstraint() + alter.ConstraintType = constraint.Contype + alter.ConstraintName = constraint.Conname + alter.IsDeferrable = constraint.Deferrable + alter.ConstraintColumns = parseColumnsFromKeys(constraint.GetKeys()) + + case pg_query.AlterTableType_AT_DisableRule: + /* + e.g. ALTER TABLE example DISABLE example_rule; + cmds:{alter_table_cmd:{subtype:AT_DisableRule name:"example_rule" behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} + checking the subType is sufficient in this case + */ + alter.RuleName = cmd.Name + //case CLUSTER ON + /* + e.g. ALTER TABLE example CLUSTER ON idx; + stmt:{alter_table_stmt:{relation:{relname:"example" inh:true relpersistence:"p" location:13} + cmds:{alter_table_cmd:{subtype:AT_ClusterOn name:"idx" behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} stmt_len:32 + + */ + } + + return alter, nil +} + +type AlterTable struct { + Query string + SchemaName string + TableName string + AlterType pg_query.AlterTableType + RuleName string + NumSetAttributes int + NumStorageOptions int + //In case AlterType - ADD_CONSTRAINT + ConstraintType pg_query.ConstrType + ConstraintName string + IsDeferrable bool + ConstraintColumns []string +} + +func (a *AlterTable) GetObjectName() string { + qualifiedTable := lo.Ternary(a.SchemaName != "", fmt.Sprintf("%s.%s", a.SchemaName, a.TableName), a.TableName) + return qualifiedTable +} +func (a *AlterTable) GetSchemaName() string { return a.SchemaName } + +func (a *AlterTable) AddPrimaryKeyOrUniqueCons() bool { + return a.ConstraintType == PRIMARY_CONSTR_TYPE || a.ConstraintType == UNIQUE_CONSTR_TYPE +} + +//===========POLICY PROCESSOR ================================ + +// PolicyProcessor handles parsing CREATE POLICY statements +type PolicyProcessor struct{} + +func NewPolicyProcessor() *PolicyProcessor { + return &PolicyProcessor{} +} + +func (policyProcessor *PolicyProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + policyNode, ok := getPolicyStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE POLICY statement") + } + + policy := &Policy{ + PolicyName: policyNode.CreatePolicyStmt.GetPolicyName(), + SchemaName: policyNode.CreatePolicyStmt.GetTable().GetSchemaname(), + TableName: policyNode.CreatePolicyStmt.GetTable().GetRelname(), + RoleNames: make([]string, 0), + } + roles := policyNode.CreatePolicyStmt.GetRoles() + /* + e.g. CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); + stmt:{create_policy_stmt:{policy_name:"p" table:{relname:"tbl1" inh:true relpersistence:"p" location:20} cmd_name:"all" + permissive:true roles:{role_spec:{roletype:ROLESPEC_CSTRING rolename:"regress_rls_eve" location:28}} roles:{role_spec: + {roletype:ROLESPEC_CSTRING rolename:"regress_rls_frank" location:45}} qual:{a_const:{boolval:{boolval:true} location:70}}}} + stmt_len:75 + + here role_spec of each roles is managing the roles related information in a POLICY DDL if any, so we can just check if there is + a role name available in it which means there is a role associated with this DDL. Hence report it. + + */ + for _, role := range roles { + roleName := role.GetRoleSpec().GetRolename() // only in case there is role associated with a policy it will error out in schema migration + if roleName != "" { + //this means there is some role or grants used in this Policy, so detecting it + policy.RoleNames = append(policy.RoleNames, roleName) + } + } + return policy, nil +} + +type Policy struct { + SchemaName string + TableName string + PolicyName string + RoleNames []string +} + +func (p *Policy) GetObjectName() string { + qualifiedTable := lo.Ternary(p.SchemaName != "", fmt.Sprintf("%s.%s", p.SchemaName, p.TableName), p.TableName) + return fmt.Sprintf("%s ON %s", p.PolicyName, qualifiedTable) +} +func (p *Policy) GetSchemaName() string { return p.SchemaName } + +//=====================TRIGGER PROCESSOR ================== + +// TriggerProcessor handles parsing CREATE Trigger statements +type TriggerProcessor struct{} + +func NewTriggerProcessor() *TriggerProcessor { + return &TriggerProcessor{} +} + +/* +e.g.CREATE CONSTRAINT TRIGGER some_trig + + AFTER DELETE ON xyz_schema.abc + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW EXECUTE PROCEDURE xyz_schema.some_trig(); + +create_trig_stmt:{isconstraint:true trigname:"some_trig" relation:{schemaname:"xyz_schema" relname:"abc" inh:true relpersistence:"p" +location:56} funcname:{string:{sval:"xyz_schema"}} funcname:{string:{sval:"some_trig"}} row:true events:8 deferrable:true initdeferred:true}} +stmt_len:160} + +e.g. CREATE TRIGGER projects_loose_fk_trigger + + AFTER DELETE ON public.projects + REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE FUNCTION xyz_schema.some_trig(); + +stmt:{create_trig_stmt:{trigname:"projects_loose_fk_trigger" relation:{schemaname:"public" relname:"projects" inh:true +relpersistence:"p" location:58} funcname:{string:{sval:"xyz_schema"}} funcname:{string:{sval:"some_trig"}} events:8 +transition_rels:{trigger_transition:{name:"old_table" is_table:true}}}} stmt_len:167} +*/ +func (triggerProcessor *TriggerProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + triggerNode, ok := getCreateTriggerStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE TRIGGER statement") + } + + trigger := &Trigger{ + SchemaName: triggerNode.CreateTrigStmt.Relation.Schemaname, + TableName: triggerNode.CreateTrigStmt.Relation.Relname, + TriggerName: triggerNode.CreateTrigStmt.Trigname, + + IsConstraint: triggerNode.CreateTrigStmt.Isconstraint, + NumTransitionRelations: len(triggerNode.CreateTrigStmt.GetTransitionRels()), + Timing: triggerNode.CreateTrigStmt.Timing, + Events: triggerNode.CreateTrigStmt.Events, + ForEachRow: triggerNode.CreateTrigStmt.Row, + } + + return trigger, nil +} + +type Trigger struct { + SchemaName string + TableName string + TriggerName string + IsConstraint bool + NumTransitionRelations int + ForEachRow bool + Timing int32 + Events int32 +} + +func (t *Trigger) GetObjectName() string { + return fmt.Sprintf("%s ON %s", t.TriggerName, t.GetTableName()) +} + +func (t *Trigger) GetTableName() string { + return lo.Ternary(t.SchemaName != "", fmt.Sprintf("%s.%s", t.SchemaName, t.TableName), t.TableName) +} + +func (t *Trigger) GetSchemaName() string { return t.SchemaName } + +/* +e.g.CREATE TRIGGER after_insert_or_delete_trigger + + BEFORE INSERT OR DELETE ON main_table + FOR EACH ROW + EXECUTE FUNCTION handle_insert_or_delete(); + +stmt:{create_trig_stmt:{trigname:"after_insert_or_delete_trigger" relation:{relname:"main_table" inh:true relpersistence:"p" +location:111} funcname:{string:{sval:"handle_insert_or_delete"}} row:true timing:2 events:12}} stmt_len:177} + +here, +timing - bits of BEFORE/AFTER/INSTEAD +events - bits of "OR" INSERT/UPDATE/DELETE/TRUNCATE +row - FOR EACH ROW (true), FOR EACH STATEMENT (false) +refer - https://github.com/pganalyze/pg_query_go/blob/c3a818d346a927c18469460bb18acb397f4f4301/parser/include/postgres/catalog/pg_trigger_d.h#L49 + + TRIGGER_TYPE_BEFORE (1 << 1) + TRIGGER_TYPE_INSERT (1 << 2) + TRIGGER_TYPE_DELETE (1 << 3) + TRIGGER_TYPE_UPDATE (1 << 4) + TRIGGER_TYPE_TRUNCATE (1 << 5) + TRIGGER_TYPE_INSTEAD (1 << 6) +*/ +func (t *Trigger) IsBeforeRowTrigger() bool { + isSecondBitSet := t.Timing&(1<<1) != 0 + return t.ForEachRow && isSecondBitSet +} + +// ========================TYPE PROCESSOR====================== + +type TypeProcessor struct{} + +func NewTypeProcessor() *TypeProcessor { + return &TypeProcessor{} +} + +func (typeProcessor *TypeProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + compositeNode, isComposite := getCompositeTypeStmtNode(parseTree) + enumNode, isEnum := getEnumTypeStmtNode(parseTree) + + switch { + case isComposite: + createType := &CreateType{ + TypeName: compositeNode.CompositeTypeStmt.Typevar.GetRelname(), + SchemaName: compositeNode.CompositeTypeStmt.Typevar.GetSchemaname(), + } + return createType, nil + case isEnum: + typeNames := enumNode.CreateEnumStmt.GetTypeName() + typeName, typeSchemaName := getTypeNameAndSchema(typeNames) + createType := &CreateType{ + TypeName: typeName, + SchemaName: typeSchemaName, + IsEnum: true, + } + return createType, nil + + default: + return nil, fmt.Errorf("not CREATE TYPE statement") + } + +} + +type CreateType struct { + TypeName string + SchemaName string + IsEnum bool +} + +func (c *CreateType) GetObjectName() string { + return lo.Ternary(c.SchemaName != "", fmt.Sprintf("%s.%s", c.SchemaName, c.TypeName), c.TypeName) +} +func (c *CreateType) GetSchemaName() string { return c.SchemaName } + +//=============================No-Op PROCESSOR ================== + +//No op parser for objects we don't have parser yet + +type NoOpProcessor struct{} + +func NewNoOpProcessor() *NoOpProcessor { + return &NoOpProcessor{} +} + +type Object struct { + ObjectName string + SchemaName string +} + +func (o *Object) GetObjectName() string { return o.ObjectName } +func (o *Object) GetSchemaName() string { return o.SchemaName } + +func (n *NoOpProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + return &Object{}, nil +} + +func GetDDLProcessor(parseTree *pg_query.ParseResult) (DDLProcessor, error) { + stmtType := GetStatementType(parseTree.Stmts[0].Stmt.ProtoReflect()) + switch stmtType { + case PG_QUERY_CREATE_STMT: + return NewTableProcessor(), nil + case PG_QUERY_INDEX_STMT: + return NewIndexProcessor(), nil + case PG_QUERY_ALTER_TABLE_STMT: + return NewAlterTableProcessor(), nil + case PG_QUERY_POLICY_STMT: + return NewPolicyProcessor(), nil + case PG_QUERY_CREATE_TRIG_STMT: + return NewTriggerProcessor(), nil + case PG_QUERY_COMPOSITE_TYPE_STMT, PG_QUERY_ENUM_TYPE_STMT: + return NewTypeProcessor(), nil + case PG_QUERY_FOREIGN_TABLE_STMT: + return NewForeignTableProcessor(), nil + default: + return NewNoOpProcessor(), nil + } +} + +const ( + ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint + SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions + DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule + CLUSTER_ON = pg_query.AlterTableType_AT_ClusterOn + EXCLUSION_CONSTR_TYPE = pg_query.ConstrType_CONSTR_EXCLUSION + FOREIGN_CONSTR_TYPE = pg_query.ConstrType_CONSTR_FOREIGN + DEFAULT_SORTING_ORDER = pg_query.SortByDir_SORTBY_DEFAULT + PRIMARY_CONSTR_TYPE = pg_query.ConstrType_CONSTR_PRIMARY + UNIQUE_CONSTR_TYPE = pg_query.ConstrType_CONSTR_UNIQUE + LIST_PARTITION = pg_query.PartitionStrategy_PARTITION_STRATEGY_LIST + PG_QUERY_CREATE_STMT = "pg_query.CreateStmt" + PG_QUERY_INDEX_STMT = "pg_query.IndexStmt" + PG_QUERY_ALTER_TABLE_STMT = "pg_query.AlterTableStmt" + PG_QUERY_POLICY_STMT = "pg_query.CreatePolicyStmt" + PG_QUERY_CREATE_TRIG_STMT = "pg_query.CreateTrigStmt" + PG_QUERY_COMPOSITE_TYPE_STMT = "pg_query.CompositeTypeStmt" + PG_QUERY_ENUM_TYPE_STMT = "pg_query.CreateEnumStmt" + PG_QUERY_FOREIGN_TABLE_STMT = "pg_query.CreateForeignTableStmt" +) + +var deferrableConstraintsList = []pg_query.ConstrType{ + pg_query.ConstrType_CONSTR_ATTR_DEFERRABLE, + pg_query.ConstrType_CONSTR_ATTR_DEFERRED, + pg_query.ConstrType_CONSTR_ATTR_IMMEDIATE, +} diff --git a/yb-voyager/src/queryparser/helpers.go b/yb-voyager/src/queryparser/helpers_protomsg.go similarity index 97% rename from yb-voyager/src/queryparser/helpers.go rename to yb-voyager/src/queryparser/helpers_protomsg.go index 09c999c80b..3403161aa8 100644 --- a/yb-voyager/src/queryparser/helpers.go +++ b/yb-voyager/src/queryparser/helpers_protomsg.go @@ -308,6 +308,21 @@ func getOneofActiveField(msg protoreflect.Message, oneofName string) protoreflec return msg.WhichOneof(oneofDescriptor) } +func GetStatementType(msg protoreflect.Message) string { + nodeMsg := getOneofActiveField(msg, "node") + if nodeMsg == nil { + return "" + } + + // Get the message corresponding to the set field + nodeValue := msg.Get(nodeMsg) + node := nodeValue.Message() + if node == nil || !node.IsValid() { + return "" + } + return GetMsgFullName(node) +} + // == Generic helper functions == // GetStringField retrieves a string field from a message. diff --git a/yb-voyager/src/queryparser/helpers_struct.go b/yb-voyager/src/queryparser/helpers_struct.go new file mode 100644 index 0000000000..591b0631b6 --- /dev/null +++ b/yb-voyager/src/queryparser/helpers_struct.go @@ -0,0 +1,290 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryparser + +import ( + "fmt" + "strings" + + pg_query "github.com/pganalyze/pg_query_go/v5" + "github.com/samber/lo" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func DeparseSelectStmt(selectStmt *pg_query.SelectStmt) (string, error) { + if selectStmt != nil { + parseResult := &pg_query.ParseResult{ + Stmts: []*pg_query.RawStmt{ + { + Stmt: &pg_query.Node{ + Node: &pg_query.Node_SelectStmt{SelectStmt: selectStmt}, + }, + }, + }, + } + + // Deparse the SelectStmt to get the string representation + selectSQL, err := pg_query.Deparse(parseResult) + return selectSQL, err + } + return "", nil +} + +func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { + return parseTree.Stmts[0].Stmt.ProtoReflect() +} + +func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { + // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE + _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) + return isPlPgSQLObject +} + +func IsViewObject(parseTree *pg_query.ParseResult) bool { + _, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) + return isViewStmt +} + +func IsMviewObject(parseTree *pg_query.ParseResult) bool { + createAsNode, isCreateAsStmt := getCreateTableAsStmtNode(parseTree) //for MVIEW case + return isCreateAsStmt && createAsNode.CreateTableAsStmt.Objtype == pg_query.ObjectType_OBJECT_MATVIEW +} + +func GetSelectStmtQueryFromViewOrMView(parseTree *pg_query.ParseResult) (string, error) { + viewNode, isViewStmt := getCreateViewNode(parseTree) + createAsNode, _ := getCreateTableAsStmtNode(parseTree) //For MVIEW case + var selectStmt *pg_query.SelectStmt + if isViewStmt { + selectStmt = viewNode.ViewStmt.GetQuery().GetSelectStmt() + } else { + selectStmt = createAsNode.CreateTableAsStmt.GetQuery().GetSelectStmt() + } + selectStmtQuery, err := DeparseSelectStmt(selectStmt) + if err != nil { + return "", fmt.Errorf("deparsing the select stmt: %v", err) + } + return selectStmtQuery, nil +} + +func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { + createFuncNode, isCreateFunc := getCreateFuncStmtNode(parseTree) + viewNode, isViewStmt := getCreateViewNode(parseTree) + createAsNode, _ := getCreateTableAsStmtNode(parseTree) + createTableNode, isCreateTable := getCreateTableStmtNode(parseTree) + createIndexNode, isCreateIndex := getCreateIndexStmtNode(parseTree) + alterTableNode, isAlterTable := getAlterStmtNode(parseTree) + switch true { + case isCreateFunc: + /* + version:160001 stmts:{stmt:{create_function_stmt:{replace:true funcname:{string:{sval:"public"}} funcname:{string:{sval:"add_employee"}} + parameters:{function_parameter:{name:"emp_name" arg_type:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"varchar"}} + typemod:-1 location:62} mode:FUNC_PARAM_DEFAULT}} parameters:{funct ... + + version:160001 stmts:{stmt:{create_function_stmt:{is_procedure:true replace:true funcname:{string:{sval:"public"}} + funcname:{string:{sval:"add_employee"}} parameters:{function_parameter:{name:"emp_name" arg_type:{names:{string:{sval:"pg_catalog"}} + names:{string:{sval:"varchar"}} typemod:-1 location:63} mode:FUNC_PARAM_DEFAULT}} ... + */ + stmt := createFuncNode.CreateFunctionStmt + objectType := "FUNCTION" + if stmt.IsProcedure { + objectType = "PROCEDURE" + } + funcNameList := stmt.GetFuncname() + return objectType, getFunctionObjectName(funcNameList) + case isViewStmt: + viewName := viewNode.ViewStmt.View + return "VIEW", getObjectNameFromRangeVar(viewName) + case IsMviewObject(parseTree): + intoMview := createAsNode.CreateTableAsStmt.Into.Rel + return "MVIEW", getObjectNameFromRangeVar(intoMview) + case isCreateTable: + return "TABLE", getObjectNameFromRangeVar(createTableNode.CreateStmt.Relation) + case isAlterTable: + return "TABLE", getObjectNameFromRangeVar(alterTableNode.AlterTableStmt.Relation) + case isCreateIndex: + indexName := createIndexNode.IndexStmt.Idxname + schemaName := createIndexNode.IndexStmt.Relation.GetSchemaname() + tableName := createIndexNode.IndexStmt.Relation.GetRelname() + fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) + displayObjName := fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName) + return "INDEX", displayObjName + default: + panic("unsupported type of parseResult") + } +} + +func isArrayType(typeName *pg_query.TypeName) bool { + return len(typeName.GetArrayBounds()) > 0 +} + +// Range Var is the struct to get the relation information like relation name, schema name, persisted relation or not, etc.. +func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string { + schema := obj.Schemaname + name := obj.Relname + return lo.Ternary(schema != "", fmt.Sprintf("%s.%s", schema, name), name) +} + +func getFunctionObjectName(funcNameList []*pg_query.Node) string { + funcName := "" + funcSchemaName := "" + if len(funcNameList) > 0 { + funcName = funcNameList[len(funcNameList)-1].GetString_().Sval // func name can be qualified / unqualifed or native / non-native proper func name will always be available at last index + } + if len(funcNameList) >= 2 { // Names list will have all the parts of qualified func name + funcSchemaName = funcNameList[len(funcNameList)-2].GetString_().Sval // // func name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index + } + return lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) +} + +func getTypeNameAndSchema(typeNames []*pg_query.Node) (string, string) { + typeName := "" + typeSchemaName := "" + if len(typeNames) > 0 { + typeName = typeNames[len(typeNames)-1].GetString_().Sval // type name can be qualified / unqualifed or native / non-native proper type name will always be available at last index + } + if len(typeNames) >= 2 { // Names list will have all the parts of qualified type name + typeSchemaName = typeNames[len(typeNames)-2].GetString_().Sval // // type name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index + } + + return typeName, typeSchemaName +} + +func getCreateTableAsStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateTableAsStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) + return node, ok +} + +func getCreateViewNode(parseTree *pg_query.ParseResult) (*pg_query.Node_ViewStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) + return node, ok +} + +func getCreateFuncStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateFunctionStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) + return node, ok +} + +func getCreateTableStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateStmt) + return node, ok +} + +func getCreateIndexStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_IndexStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_IndexStmt) + return node, ok +} + +func getAlterStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_AlterTableStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_AlterTableStmt) + return node, ok +} + +func getCreateTriggerStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateTrigStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTrigStmt) + return node, ok +} + +func getPolicyStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreatePolicyStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreatePolicyStmt) + return node, ok +} + +func getCompositeTypeStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CompositeTypeStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CompositeTypeStmt) + return node, ok +} + +func getEnumTypeStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateEnumStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateEnumStmt) + return node, ok +} +func getForeignTableStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateForeignTableStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateForeignTableStmt) + return node, ok +} + +func IsFunctionObject(parseTree *pg_query.ParseResult) bool { + funcNode, ok := getCreateFuncStmtNode(parseTree) + if !ok { + return false + } + return !funcNode.CreateFunctionStmt.IsProcedure +} + +/* +return type ex- +CREATE OR REPLACE FUNCTION public.process_combined_tbl( + + ... + +) +RETURNS public.combined_tbl.maddr%TYPE AS +return_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} names:{string:{sval:"maddr"}} +pct_type:true typemod:-1 location:226} +*/ +func GetReturnTypeOfFunc(parseTree *pg_query.ParseResult) string { + funcNode, _ := getCreateFuncStmtNode(parseTree) + returnType := funcNode.CreateFunctionStmt.GetReturnType() + return convertParserTypeNameToString(returnType) +} + +func getQualifiedTypeName(typeNames []*pg_query.Node) string { + var typeNameStrings []string + for _, n := range typeNames { + typeNameStrings = append(typeNameStrings, n.GetString_().Sval) + } + return strings.Join(typeNameStrings, ".") +} + +func convertParserTypeNameToString(typeVar *pg_query.TypeName) string { + typeNames := typeVar.GetNames() + finalTypeName := getQualifiedTypeName(typeNames) // type name can qualified table_name.column in case of %TYPE + if typeVar.PctType { // %TYPE declaration, so adding %TYPE for using it further + return finalTypeName + "%TYPE" + } + return finalTypeName +} + +/* +function ex - +CREATE OR REPLACE FUNCTION public.process_combined_tbl( + + p_id int, + p_c public.combined_tbl.c%TYPE, + p_bitt public.combined_tbl.bitt%TYPE, + .. + +) +parseTree- +parameters:{function_parameter:{name:"p_id" arg_type:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:66} +mode:FUNC_PARAM_DEFAULT}} parameters:{function_parameter:{name:"p_c" arg_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} +names:{string:{sval:"c"}} pct_type:true typemod:-1 location:87} mode:FUNC_PARAM_DEFAULT}} parameters:{function_parameter:{name:"p_bitt" +arg_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} names:{string:{sval:"bitt"}} pct_type:true typemod:-1 +location:136} mode:FUNC_PARAM_DEFAULT}} +*/ +func GetFuncParametersTypeNames(parseTree *pg_query.ParseResult) []string { + funcNode, _ := getCreateFuncStmtNode(parseTree) + parameters := funcNode.CreateFunctionStmt.GetParameters() + var paramTypeNames []string + for _, param := range parameters { + funcParam, ok := param.Node.(*pg_query.Node_FunctionParameter) + if ok { + paramType := funcParam.FunctionParameter.ArgType + paramTypeNames = append(paramTypeNames, convertParserTypeNameToString(paramType)) + } + } + return paramTypeNames +} diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/queryparser/query_parser.go index e8392092b9..6e6c4149a1 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/queryparser/query_parser.go @@ -17,12 +17,9 @@ package queryparser import ( "fmt" - "strings" pg_query "github.com/pganalyze/pg_query_go/v5" - "github.com/samber/lo" log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/reflect/protoreflect" ) func Parse(query string) (*pg_query.ParseResult, error) { @@ -45,197 +42,16 @@ func ParsePLPGSQLToJson(query string) (string, error) { return jsonString, err } -func DeparseSelectStmt(selectStmt *pg_query.SelectStmt) (string, error) { - if selectStmt != nil { - parseResult := &pg_query.ParseResult{ - Stmts: []*pg_query.RawStmt{ - { - Stmt: &pg_query.Node{ - Node: &pg_query.Node_SelectStmt{SelectStmt: selectStmt}, - }, - }, - }, - } - - // Deparse the SelectStmt to get the string representation - selectSQL, err := pg_query.Deparse(parseResult) - return selectSQL, err - } - return "", nil -} - -func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { - return parseTree.Stmts[0].Stmt.ProtoReflect() -} - -func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { - // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE - _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) - return isPlPgSQLObject -} - -func IsViewObject(parseTree *pg_query.ParseResult) bool { - _, isViewStmt := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) - return isViewStmt -} - -func IsMviewObject(parseTree *pg_query.ParseResult) bool { - createAsNode, isCreateAsStmt := getCreateTableAsStmtNode(parseTree) //for MVIEW case - return isCreateAsStmt && createAsNode.CreateTableAsStmt.Objtype == pg_query.ObjectType_OBJECT_MATVIEW -} - -func GetSelectStmtQueryFromViewOrMView(parseTree *pg_query.ParseResult) (string, error) { - viewNode, isViewStmt := getCreateViewNode(parseTree) - createAsNode, _ := getCreateTableAsStmtNode(parseTree) //For MVIEW case - var selectStmt *pg_query.SelectStmt - if isViewStmt { - selectStmt = viewNode.ViewStmt.GetQuery().GetSelectStmt() - } else { - selectStmt = createAsNode.CreateTableAsStmt.GetQuery().GetSelectStmt() - } - selectStmtQuery, err := DeparseSelectStmt(selectStmt) +func ProcessDDL(parseTree *pg_query.ParseResult) (DDLObject, error) { + processor, err := GetDDLProcessor(parseTree) if err != nil { - return "", fmt.Errorf("deparsing the select stmt: %v", err) + return nil, fmt.Errorf("getting processor failed: %v", err) } - return selectStmtQuery, nil -} - -func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { - createFuncNode, isCreateFunc := getCreateFuncStmtNode(parseTree) - viewNode, isViewStmt := getCreateViewNode(parseTree) - createAsNode, _ := getCreateTableAsStmtNode(parseTree) - switch true { - case isCreateFunc: - /* - version:160001 stmts:{stmt:{create_function_stmt:{replace:true funcname:{string:{sval:"public"}} funcname:{string:{sval:"add_employee"}} - parameters:{function_parameter:{name:"emp_name" arg_type:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"varchar"}} - typemod:-1 location:62} mode:FUNC_PARAM_DEFAULT}} parameters:{funct ... - - version:160001 stmts:{stmt:{create_function_stmt:{is_procedure:true replace:true funcname:{string:{sval:"public"}} - funcname:{string:{sval:"add_employee"}} parameters:{function_parameter:{name:"emp_name" arg_type:{names:{string:{sval:"pg_catalog"}} - names:{string:{sval:"varchar"}} typemod:-1 location:63} mode:FUNC_PARAM_DEFAULT}} ... - */ - stmt := createFuncNode.CreateFunctionStmt - objectType := "FUNCTION" - if stmt.IsProcedure { - objectType = "PROCEDURE" - } - funcNameList := stmt.GetFuncname() - return objectType, getFunctionObjectName(funcNameList) - case isViewStmt: - viewName := viewNode.ViewStmt.View - return "VIEW", getObjectNameFromRangeVar(viewName) - case IsMviewObject(parseTree): - intoMview := createAsNode.CreateTableAsStmt.Into.Rel - return "MVIEW", getObjectNameFromRangeVar(intoMview) - default: - panic("unsupported type of parseResult") - } -} - -// Range Var is the struct to get the relation information like relation name, schema name, persisted relation or not, etc.. -func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string { - schema := obj.Schemaname - name := obj.Relname - return lo.Ternary(schema != "", fmt.Sprintf("%s.%s", schema, name), name) -} - -func getFunctionObjectName(funcNameList []*pg_query.Node) string { - funcName := "" - funcSchemaName := "" - if len(funcNameList) > 0 { - funcName = funcNameList[len(funcNameList)-1].GetString_().Sval // func name can be qualified / unqualifed or native / non-native proper func name will always be available at last index - } - if len(funcNameList) >= 2 { // Names list will have all the parts of qualified func name - funcSchemaName = funcNameList[len(funcNameList)-2].GetString_().Sval // // func name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index - } - return lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) -} - -func getCreateTableAsStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateTableAsStmt, bool) { - node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateTableAsStmt) - return node, ok -} -func getCreateViewNode(parseTree *pg_query.ParseResult) (*pg_query.Node_ViewStmt, bool) { - node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_ViewStmt) - return node, ok -} - -func getCreateFuncStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateFunctionStmt, bool) { - node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateFunctionStmt) - return node, ok -} - -func IsFunctionObject(parseTree *pg_query.ParseResult) bool { - funcNode, ok := getCreateFuncStmtNode(parseTree) - if !ok { - return false - } - return !funcNode.CreateFunctionStmt.IsProcedure -} - -/* -return type ex- -CREATE OR REPLACE FUNCTION public.process_combined_tbl( - - ... - -) -RETURNS public.combined_tbl.maddr%TYPE AS -return_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} names:{string:{sval:"maddr"}} -pct_type:true typemod:-1 location:226} -*/ -func GetReturnTypeOfFunc(parseTree *pg_query.ParseResult) string { - funcNode, _ := getCreateFuncStmtNode(parseTree) - returnType := funcNode.CreateFunctionStmt.GetReturnType() - return convertParserTypeNameToString(returnType) -} - -func getQualifiedTypeName(typeNames []*pg_query.Node) string { - var typeNameStrings []string - for _, n := range typeNames { - typeNameStrings = append(typeNameStrings, n.GetString_().Sval) - } - return strings.Join(typeNameStrings, ".") -} - -func convertParserTypeNameToString(typeVar *pg_query.TypeName) string { - typeNames := typeVar.GetNames() - finalTypeName := getQualifiedTypeName(typeNames) // type name can qualified table_name.column in case of %TYPE - if typeVar.PctType { // %TYPE declaration, so adding %TYPE for using it further - return finalTypeName + "%TYPE" + ddlObject, err := processor.Process(parseTree) + if err != nil { + return nil, fmt.Errorf("parsing DDL failed: %v", err) } - return finalTypeName -} -/* -function ex - -CREATE OR REPLACE FUNCTION public.process_combined_tbl( - - p_id int, - p_c public.combined_tbl.c%TYPE, - p_bitt public.combined_tbl.bitt%TYPE, - .. - -) -parseTree- -parameters:{function_parameter:{name:"p_id" arg_type:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 location:66} -mode:FUNC_PARAM_DEFAULT}} parameters:{function_parameter:{name:"p_c" arg_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} -names:{string:{sval:"c"}} pct_type:true typemod:-1 location:87} mode:FUNC_PARAM_DEFAULT}} parameters:{function_parameter:{name:"p_bitt" -arg_type:{names:{string:{sval:"public"}} names:{string:{sval:"combined_tbl"}} names:{string:{sval:"bitt"}} pct_type:true typemod:-1 -location:136} mode:FUNC_PARAM_DEFAULT}} -*/ -func GetFuncParametersTypeNames(parseTree *pg_query.ParseResult) []string { - funcNode, _ := getCreateFuncStmtNode(parseTree) - parameters := funcNode.CreateFunctionStmt.GetParameters() - var paramTypeNames []string - for _, param := range parameters { - funcParam, ok := param.Node.(*pg_query.Node_FunctionParameter) - if ok { - paramType := funcParam.FunctionParameter.ArgType - paramTypeNames = append(paramTypeNames, convertParserTypeNameToString(paramType)) - } - } - return paramTypeNames + return ddlObject, nil } From 8e58732a8977eb580eec0ecbcedb72c55af6ad4b Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Thu, 5 Dec 2024 16:54:00 +0530 Subject: [PATCH 034/105] Reworded PLPGSQL to PL/pgSQL in the description of unsupported PL/pgSQL objects (#2038) --- yb-voyager/cmd/constants.go | 2 +- yb-voyager/cmd/templates/migration_assessment_report.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 5b19baaa82..85c9f225e4 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -182,7 +182,7 @@ const ( DATATYPE_ISSUE_TYPE_DESCRIPTION = "Data types of the source database that are not supported on the target YugabyteDB." MIGRATION_CAVEATS_TYPE_DESCRIPTION = "Migration Caveats highlights the current limitations with the migration workflow." UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION = "Source database queries not supported in YugabyteDB, identified by scanning system tables." - UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PLPGSQL code block" + UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PL/pgSQL code block" SCHEMA_SUMMARY_DESCRIPTION = "Objects that will be created on the target YugabyteDB." SCHEMA_SUMMARY_DESCRIPTION_ORACLE = SCHEMA_SUMMARY_DESCRIPTION + " Some of the index and sequence names might be different from those in the source database." diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 85a10b1446..67d95e33c1 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -311,7 +311,7 @@

    Unsupported PL/pgSQL objects

    {{ if .UnsupportedPlPgSqlObjects}} -

    Source schema objects having unsupported statements in PLPGSQL code block:

    +

    Source schema objects having unsupported statements in PL/pgSQL code block:

    Voyager Version{{ .VoyagerVersion }}
    Target DB Version{{ .TargetDBVersion }}
    Database Name{{ .SchemaSummary.DBName }}
    Schema Name{{ join .SchemaSummary.SchemaNames ", " }}
    From 31d6d9cce03fcd86d98e0598497bb4fa6675669e Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Thu, 5 Dec 2024 16:56:58 +0530 Subject: [PATCH 035/105] Added the NOCACHE clause to the sequences tables in Oracle automation (#2037) --- .../oracle/sequences/sequence_schema.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/migtests/tests/oracle/sequences/sequence_schema.sql b/migtests/tests/oracle/sequences/sequence_schema.sql index f3c34ea7b8..1baac1c7fc 100644 --- a/migtests/tests/oracle/sequences/sequence_schema.sql +++ b/migtests/tests/oracle/sequences/sequence_schema.sql @@ -1,70 +1,70 @@ drop table identity_demo_generated_always; CREATE TABLE identity_demo_generated_always ( - id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) NOT NULL ); drop table identity_demo_generated_by_def; CREATE TABLE identity_demo_generated_by_def ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); DROP TABLE identity_demo_with_null; CREATE TABLE identity_demo_with_null ( - id NUMBER GENERATED BY DEFAULT ON NULL AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT ON NULL AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); drop table identity_demo_generated_always_start_with; CREATE TABLE identity_demo_generated_always_start_with ( - id NUMBER GENERATED ALWAYS AS IDENTITY start with 101 PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE start with 101 PRIMARY KEY, description VARCHAR2(100) NOT NULL ); drop table identity_demo_generated_by_def_start_with; CREATE TABLE identity_demo_generated_by_def_start_with ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY start with 101 PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE start with 101 PRIMARY KEY, description VARCHAR2(100) not null ); drop table identity_demo_generated_by_def_inc_by; CREATE TABLE identity_demo_generated_by_def_inc_by ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY increment by 101 PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE increment by 101 PRIMARY KEY, description VARCHAR2(100) not null ); drop table identity_demo_generated_by_def_st_with_inc_by; CREATE TABLE identity_demo_generated_by_def_st_with_inc_by ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY start with 5 increment by 101 PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE start with 5 increment by 101 PRIMARY KEY, description VARCHAR2(100) not null ); drop table empty_identity_always; CREATE TABLE empty_identity_always ( - id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) NOT NULL ); drop table empty_identity_def; CREATE TABLE empty_identity_def ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); drop table "Case_Sensitive_always"; CREATE TABLE "Case_Sensitive_always" ( - id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); From fb5249e2d6381d478784aadfeae75aa5de88e801 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:31:35 +0530 Subject: [PATCH 036/105] Removed the input of original owner of tables from user in pg grant script (#2031) Modified the permission grant scrip and removed the parameter to enter the original_owner_of_tables. - Now we fetch the owner of the tables automatically, we select all the DISTINCT owners of all the tables in the schema_list - We run a loop over these owners and run the grant statement to grant the replication group to these owners. --- ...voyager-pg-grant-migration-permissions.sql | 41 +++++++++++++------ migtests/scripts/functions.sh | 4 +- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql b/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql index ec5c08f4c1..81efa5c60c 100644 --- a/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql +++ b/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql @@ -1,8 +1,8 @@ --- How to use the script: -- Run the script with psql command line tool, passing the necessary parameters: ---- psql -h -d -U -v voyager_user='' -v schema_list='' -v is_live_migration= -v is_live_migration_fall_back= -v replication_group='' -v original_owner_of_tables='' -f +--- psql -h -d -U -v voyager_user='' -v schema_list='' -v is_live_migration= -v is_live_migration_fall_back= -v replication_group='' -f --- Example: ---- psql -h -d -U -v voyager_user='ybvoyager' -v schema_list='schema1,public,schema2' -v is_live_migration=1 -v is_live_migration_fall_back=0 -v replication_group='replication_group' -v original_owner_of_tables='postgres' -f /home/ubuntu/yb-voyager-pg-grant-migration-permissions.sql +--- psql -h -d -U -v voyager_user='ybvoyager' -v schema_list='schema1,public,schema2' -v is_live_migration=1 -v is_live_migration_fall_back=0 -v replication_group='replication_group' -f /home/ubuntu/yb-voyager-pg-grant-migration-permissions.sql --- Parameters: --- : The hostname of the PostgreSQL server. --- : The name of the database to connect to. @@ -12,7 +12,6 @@ --- : A flag indicating if this is a live migration (1 for true, 0 for false). If set to 0 then the script will check for permissions for an offline migration. --- : A flag indicating if this is a live migration with fallback (1 for true, 0 for false). If set to 0 then the script will detect permissions for live migration with fall-forward. Should only be set to 1 when is_live_migration is also set to 1. Does not need to be provided unless is_live_migration is set to 1. --- : The name of the replication group to be created. Not needed for offline migration. ---- : The original owner of the tables to be added to the replication group. Not needed for offline migration. \echo '' \echo '--- Checking Variables ---' @@ -41,7 +40,7 @@ \q \endif --- If live migration is enabled, then is_live_migration_fall_back, replication_group and original_owner_of_tables should be provided +-- If live migration is enabled, then is_live_migration_fall_back, replication_group should be provided \if :is_live_migration -- Check if is_live_migration_fall_back is provided @@ -59,14 +58,6 @@ \echo 'Error: replication_group flag is not provided!' \q \endif - - -- Check if original_owner_of_tables is provided - \if :{?original_owner_of_tables} - \echo 'Original owner of tables is provided: ':original_owner_of_tables - \else - \echo 'Error: original_owner_of_tables flag is not provided!' - \q - \endif \endif -- If live migration fallback is provided and enabled, then is_live_migration should be enabled @@ -165,7 +156,31 @@ GRANT pg_read_all_stats to :voyager_user; -- Add the original owner of the tables to the group \echo '' \echo '--- Adding Original Owner to Replication Group ---' - GRANT :replication_group TO :original_owner_of_tables; + DO $$ + DECLARE + tableowner TEXT; + schema_list TEXT[] := string_to_array(current_setting('myvars.schema_list'), ','); -- Convert the schema list to an array + replication_group TEXT := current_setting('myvars.replication_group'); -- Get the replication group from settings + BEGIN + -- Generate the GRANT statements and execute them dynamically + FOR tableowner IN + SELECT DISTINCT t.tableowner + FROM pg_catalog.pg_tables t + WHERE t.schemaname = ANY (schema_list) -- Use the schema_list variable + AND NOT EXISTS ( + SELECT 1 + FROM pg_roles r + WHERE r.rolname = t.tableowner + AND pg_has_role(t.tableowner, replication_group, 'USAGE') -- Use the replication_group variable + ) + LOOP + -- Display the GRANT statement + RAISE NOTICE 'Granting role: GRANT % TO %;', replication_group, tableowner; + + -- Execute the GRANT statement + EXECUTE format('GRANT %I TO %I;', replication_group, tableowner); + END LOOP; + END $$; -- Add the user ybvoyager to the replication group \echo '' diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index f8b707bcb3..eeab236784 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -155,7 +155,7 @@ grant_permissions_for_live_migration_pg() { db_name=$1 db_schema=$2 conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" - psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${db_schema}" -v replication_group='replication_group' -v original_owner_of_tables="${SOURCE_DB_ADMIN_USER}" -v is_live_migration=1 -v is_live_migration_fall_back=0 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql + psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${db_schema}" -v replication_group='replication_group' -v is_live_migration=1 -v is_live_migration_fall_back=0 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql } grant_permissions() { @@ -676,7 +676,7 @@ setup_fallback_environment() { run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${SOURCE_DB_NAME}" - psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v original_owner_of_tables="${SOURCE_DB_ADMIN_USER}" -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql + psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql disable_triggers_sql=$(mktemp) drop_constraints_sql=$(mktemp) From 6a1af70c8be1cfc4e6520f276a8b12705c285a2f Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 5 Dec 2024 17:39:05 +0530 Subject: [PATCH 037/105] Do not detect the DML issues on all DDL types from DDLProcessor in GetDMLIssues() (#2032) https://yugabyte.atlassian.net/browse/DB-14204 We should not detect issues on CREATE MVIEW/VIEW queries coming from pgss to GetDMLIssues as we are already detecting them in the PL/pgSQL section. - skipping the DDLs in the GetDMLIssues() function for which we have the Processor in the DDLProcessor. For now, this should be enough for our requirements. --- yb-voyager/src/queryissue/queryissue.go | 9 +- yb-voyager/src/queryissue/unsupported_ddls.go | 20 ++++ yb-voyager/src/queryparser/ddl_processor.go | 108 +++++++++++++++--- yb-voyager/src/queryparser/helpers_struct.go | 12 ++ 4 files changed, 129 insertions(+), 20 deletions(-) diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/queryissue/queryissue.go index 2c1871b637..6f9b31436c 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/queryissue/queryissue.go @@ -338,7 +338,14 @@ func (p *ParserIssueDetector) getDMLIssues(query string) ([]issue.IssueInstance, if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) } - + isDDL, err := queryparser.IsDDL(parseTree) + if err != nil { + return nil, fmt.Errorf("error checking if query is a DDL: %v", err) + } + if isDDL { + //Skip all the DDLs coming to this function + return nil, nil + } var result []issue.IssueInstance var unsupportedConstructs []string visited := make(map[protoreflect.Message]bool) diff --git a/yb-voyager/src/queryissue/unsupported_ddls.go b/yb-voyager/src/queryissue/unsupported_ddls.go index 31e9ae531a..fe4121e600 100644 --- a/yb-voyager/src/queryissue/unsupported_ddls.go +++ b/yb-voyager/src/queryissue/unsupported_ddls.go @@ -537,6 +537,22 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issu return issues, nil } +// ==============VIEW ISSUE DETECTOR ====================== + +type ViewIssueDetector struct{} + +func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + return nil, nil +} + +// ==============MVIEW ISSUE DETECTOR ====================== + +type MViewIssueDetector struct{} + +func (v *MViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { + return nil, nil +} + //=============NO-OP ISSUE DETECTOR =========================== // Need to handle all the cases for which we don't have any issues detector @@ -568,6 +584,10 @@ func (p *ParserIssueDetector) GetDDLDetector(obj queryparser.DDLObject) (DDLIssu }, nil case *queryparser.ForeignTable: return &ForeignTableIssueDetector{}, nil + case *queryparser.View: + return &ViewIssueDetector{}, nil + case *queryparser.MView: + return &MViewIssueDetector{}, nil default: return &NoOpIssueDetector{}, nil } diff --git a/yb-voyager/src/queryparser/ddl_processor.go b/yb-voyager/src/queryparser/ddl_processor.go index fb0ebf0f95..c9598667bb 100644 --- a/yb-voyager/src/queryparser/ddl_processor.go +++ b/yb-voyager/src/queryparser/ddl_processor.go @@ -808,9 +808,69 @@ func (c *CreateType) GetObjectName() string { } func (c *CreateType) GetSchemaName() string { return c.SchemaName } +//===========================VIEW PROCESSOR=================== + +type ViewProcessor struct{} + +func NewViewProcessor() *ViewProcessor { + return &ViewProcessor{} +} + +func (v *ViewProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + viewNode, ok := getCreateViewNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE VIEW statement") + } + view := View{ + SchemaName: viewNode.ViewStmt.View.Schemaname, + ViewName: viewNode.ViewStmt.View.Relname, + } + return &view, nil +} + +type View struct { + SchemaName string + ViewName string +} + +func (v *View) GetObjectName() string { + return lo.Ternary(v.SchemaName != "", fmt.Sprintf("%s.%s", v.SchemaName, v.ViewName), v.ViewName) +} +func (v *View) GetSchemaName() string { return v.SchemaName } + +//===========================MVIEW PROCESSOR=================== + +type MViewProcessor struct{} + +func NewMViewProcessor() *MViewProcessor { + return &MViewProcessor{} +} + +func (mv *MViewProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + mviewNode, ok := getCreateTableAsStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE VIEW statement") + } + mview := MView{ + SchemaName: mviewNode.CreateTableAsStmt.Into.Rel.Schemaname, + ViewName: mviewNode.CreateTableAsStmt.Into.Rel.Relname, + } + return &mview, nil +} + +type MView struct { + SchemaName string + ViewName string +} + +func (mv *MView) GetObjectName() string { + return lo.Ternary(mv.SchemaName != "", fmt.Sprintf("%s.%s", mv.SchemaName, mv.ViewName), mv.ViewName) +} +func (mv *MView) GetSchemaName() string { return mv.SchemaName } + //=============================No-Op PROCESSOR ================== -//No op parser for objects we don't have parser yet +//No op Processor for objects we don't have Processor yet type NoOpProcessor struct{} @@ -847,30 +907,40 @@ func GetDDLProcessor(parseTree *pg_query.ParseResult) (DDLProcessor, error) { return NewTypeProcessor(), nil case PG_QUERY_FOREIGN_TABLE_STMT: return NewForeignTableProcessor(), nil + case PG_QUERY_VIEW_STMT: + return NewViewProcessor(), nil + case PG_QUERY_CREATE_TABLE_AS_STMT: + if IsMviewObject(parseTree) { + return NewMViewProcessor(), nil + } else { + return NewNoOpProcessor(), nil + } default: return NewNoOpProcessor(), nil } } const ( - ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint - SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions - DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule - CLUSTER_ON = pg_query.AlterTableType_AT_ClusterOn - EXCLUSION_CONSTR_TYPE = pg_query.ConstrType_CONSTR_EXCLUSION - FOREIGN_CONSTR_TYPE = pg_query.ConstrType_CONSTR_FOREIGN - DEFAULT_SORTING_ORDER = pg_query.SortByDir_SORTBY_DEFAULT - PRIMARY_CONSTR_TYPE = pg_query.ConstrType_CONSTR_PRIMARY - UNIQUE_CONSTR_TYPE = pg_query.ConstrType_CONSTR_UNIQUE - LIST_PARTITION = pg_query.PartitionStrategy_PARTITION_STRATEGY_LIST - PG_QUERY_CREATE_STMT = "pg_query.CreateStmt" - PG_QUERY_INDEX_STMT = "pg_query.IndexStmt" - PG_QUERY_ALTER_TABLE_STMT = "pg_query.AlterTableStmt" - PG_QUERY_POLICY_STMT = "pg_query.CreatePolicyStmt" - PG_QUERY_CREATE_TRIG_STMT = "pg_query.CreateTrigStmt" - PG_QUERY_COMPOSITE_TYPE_STMT = "pg_query.CompositeTypeStmt" - PG_QUERY_ENUM_TYPE_STMT = "pg_query.CreateEnumStmt" - PG_QUERY_FOREIGN_TABLE_STMT = "pg_query.CreateForeignTableStmt" + ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint + SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions + DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule + CLUSTER_ON = pg_query.AlterTableType_AT_ClusterOn + EXCLUSION_CONSTR_TYPE = pg_query.ConstrType_CONSTR_EXCLUSION + FOREIGN_CONSTR_TYPE = pg_query.ConstrType_CONSTR_FOREIGN + DEFAULT_SORTING_ORDER = pg_query.SortByDir_SORTBY_DEFAULT + PRIMARY_CONSTR_TYPE = pg_query.ConstrType_CONSTR_PRIMARY + UNIQUE_CONSTR_TYPE = pg_query.ConstrType_CONSTR_UNIQUE + LIST_PARTITION = pg_query.PartitionStrategy_PARTITION_STRATEGY_LIST + PG_QUERY_CREATE_STMT = "pg_query.CreateStmt" + PG_QUERY_INDEX_STMT = "pg_query.IndexStmt" + PG_QUERY_ALTER_TABLE_STMT = "pg_query.AlterTableStmt" + PG_QUERY_POLICY_STMT = "pg_query.CreatePolicyStmt" + PG_QUERY_CREATE_TRIG_STMT = "pg_query.CreateTrigStmt" + PG_QUERY_COMPOSITE_TYPE_STMT = "pg_query.CompositeTypeStmt" + PG_QUERY_ENUM_TYPE_STMT = "pg_query.CreateEnumStmt" + PG_QUERY_FOREIGN_TABLE_STMT = "pg_query.CreateForeignTableStmt" + PG_QUERY_VIEW_STMT = "pg_query.ViewStmt" + PG_QUERY_CREATE_TABLE_AS_STMT = "pg_query.CreateTableAsStmt" ) var deferrableConstraintsList = []pg_query.ConstrType{ diff --git a/yb-voyager/src/queryparser/helpers_struct.go b/yb-voyager/src/queryparser/helpers_struct.go index 591b0631b6..e332547cd7 100644 --- a/yb-voyager/src/queryparser/helpers_struct.go +++ b/yb-voyager/src/queryparser/helpers_struct.go @@ -288,3 +288,15 @@ func GetFuncParametersTypeNames(parseTree *pg_query.ParseResult) []string { } return paramTypeNames } + + +func IsDDL(parseTree *pg_query.ParseResult) (bool, error) { + ddlParser, err := GetDDLProcessor(parseTree) + if err != nil { + return false, fmt.Errorf("error getting a ddl parser: %w", err) + } + _, ok := ddlParser.(*NoOpProcessor) + //Considering all the DDLs we have a Processor for as of now. + //Not Full-proof as we don't have all DDL types but atleast we will skip all the types we know currently + return !ok, nil +} \ No newline at end of file From 4ab55b38e44696ab7a5953c34434d5076d9b6c09 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 5 Dec 2024 17:41:36 +0530 Subject: [PATCH 038/105] Create NOT VALID constraints only in post-snapshot-import mode with import-schema (#2036) Fixing an issue where the presence of NOT VALID constraints in the schema can constraint violation errors in import-data. https://yugabyte.atlassian.net/browse/DB-13881 Fix: Creating the NOT VALID constraints during post-snapshot-import mode when the data is already loaded which will not lead to any constraint violation errors. Added tests in PG and Oracle's constraints test. --- .../constraints/constraints_schema_data.sql | 17 +++---- migtests/tests/oracle/constraints/validate | 6 ++- .../constraints/pg_constraints_automation.sql | 16 ++++--- migtests/tests/pg/constraints/validate | 6 ++- yb-voyager/cmd/importSchema.go | 10 +++- yb-voyager/cmd/importSchemaYugabyteDB.go | 47 +++++++++++++++++-- yb-voyager/src/queryparser/ddl_processor.go | 19 ++++++-- 7 files changed, 95 insertions(+), 26 deletions(-) diff --git a/migtests/tests/oracle/constraints/constraints_schema_data.sql b/migtests/tests/oracle/constraints/constraints_schema_data.sql index 84cd2ef524..3cc1d1d9ef 100644 --- a/migtests/tests/oracle/constraints/constraints_schema_data.sql +++ b/migtests/tests/oracle/constraints/constraints_schema_data.sql @@ -41,17 +41,18 @@ CREATE TABLE check_test ( ID int GENERATED BY DEFAULT AS IDENTITY, first_name varchar(255) NOT NULL, last_name varchar(255), + middle_name varchar(255) NOT NULL, Age int, CHECK (Age>=18) ); -insert into check_test (first_name, last_name, age) values ('Modestine', 'MacMeeking', 20); -insert into check_test (first_name, last_name, age) values ('Genna', 'Kaysor', 50); -insert into check_test (first_name, last_name, age) values ('Tess', 'Wesker', 56); -insert into check_test (first_name, last_name, age) values ('Magnum', 'Danzelman', 89); -insert into check_test (first_name, last_name, age) values ('Mitzi', 'Pidwell', 34); -insert into check_test (first_name, last_name, age) values ('Milzie', 'Rohlfing', 70); - - +insert into check_test (first_name, middle_name, last_name, age) values ('Modestine', 'null', 'MacMeeking', 20); +insert into check_test (first_name, middle_name, last_name, age) values ('Genna', 'null', 'Kaysor', 50); +insert into check_test (first_name, middle_name, last_name, age) values ('Tess', 'null', 'Wesker', 56); +insert into check_test (first_name, middle_name, last_name, age) values ('Magnum', 'null', 'Danzelman', 89); +insert into check_test (first_name, middle_name, last_name, age) values ('Mitzi', 'null', 'Pidwell', 34); +insert into check_test (first_name, middle_name, last_name, age) values ('Milzie', 'null', 'Rohlfing', 70); + +ALTER TABLE check_test ADD CONSTRAINT novalid_con CHECK(middle_name<>'null') NOVALIDATE; drop table default_test; diff --git a/migtests/tests/oracle/constraints/validate b/migtests/tests/oracle/constraints/validate index ee964c8bc7..c5d234cb31 100755 --- a/migtests/tests/oracle/constraints/validate +++ b/migtests/tests/oracle/constraints/validate @@ -30,7 +30,11 @@ QUERIES_CHECK = { 'code': "23505" }, 'CHECK_CONDITION': { - 'query': "insert into public.check_test (id, first_name, last_name, age) values (7, 'Tom', 'Stewan', 15);", + 'query': "insert into public.check_test (id, first_name, middle_name, last_name, age) values (7, 'Tom', 'gfh', 'Stewan', 15);", + 'code': "23514" + }, + 'CHECK_CONDITION_NOT_VALID': { + 'query': "insert into public.check_test (id, first_name, middle_name, last_name, age) values (7, 'Tom', 'null', 'Stewan', 25);", 'code': "23514" }, 'FORIEGN_CHECK': { diff --git a/migtests/tests/pg/constraints/pg_constraints_automation.sql b/migtests/tests/pg/constraints/pg_constraints_automation.sql index 4f70492627..d7929263c1 100644 --- a/migtests/tests/pg/constraints/pg_constraints_automation.sql +++ b/migtests/tests/pg/constraints/pg_constraints_automation.sql @@ -39,17 +39,19 @@ drop table if exists check_test; CREATE TABLE check_test ( ID serial primary key, first_name varchar(255) NOT NULL, + middle_name varchar(255) not null, last_name varchar(255), Age int, CHECK (Age>=18) ); -insert into check_test (first_name, last_name, age) values ('Modestine', 'MacMeeking', 20); -insert into check_test (first_name, last_name, age) values ('Genna', 'Kaysor', 50); -insert into check_test (first_name, last_name, age) values ('Tess', 'Wesker', 56); -insert into check_test (first_name, last_name, age) values ('Magnum', 'Danzelman', 89); -insert into check_test (first_name, last_name, age) values ('Mitzi', 'Pidwell', 34); -insert into check_test (first_name, last_name, age) values ('Milzie', 'Rohlfing', 70); - +insert into check_test (first_name, middle_name, last_name, age) values ('Modestine', '', 'MacMeeking', 20); +insert into check_test (first_name, middle_name, last_name, age) values ('Genna', '', 'Kaysor', 50); +insert into check_test (first_name, middle_name, last_name, age) values ('Tess', '', 'Wesker', 56); +insert into check_test (first_name, middle_name, last_name, age) values ('Magnum', '', 'Danzelman', 89); +insert into check_test (first_name, middle_name, last_name, age) values ('Mitzi', '', 'Pidwell', 34); +insert into check_test (first_name, middle_name, last_name, age) values ('Milzie', '', 'Rohlfing', 70); + +ALTER TABLE check_test ADD CONSTRAINT not_valid_cons CHECK(middle_name<>'') NOT VALID; drop table if exists default_test; diff --git a/migtests/tests/pg/constraints/validate b/migtests/tests/pg/constraints/validate index 7199571d27..82d036ded2 100755 --- a/migtests/tests/pg/constraints/validate +++ b/migtests/tests/pg/constraints/validate @@ -29,7 +29,11 @@ QUERIES_CHECK = { 'code': "23505" }, 'CHECK_CONDITION': { - 'query': "insert into check_test (id, first_name, last_name, age) values (7, 'Tom', 'Stewan', 15);", + 'query': "insert into check_test (id, first_name, middle_name, last_name, age) values (7, 'Tom', 'abc', 'Stewan', 15);", + 'code': "23514" + }, + 'CHECK_CONDITION_NOT_VALID': { + 'query': "insert into check_test (id, first_name, middle_name, last_name, age) values (7, 'Tom', '', 'Stewan', 52);", 'code': "23514" }, 'FOREIGN_CHECK': { diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index 74214c1d9b..a3e154688b 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -225,8 +225,14 @@ func importSchema() error { dumpStatements(finalFailedSqlStmts, filepath.Join(exportDir, "schema", "failed.sql")) } - if flagPostSnapshotImport && flagRefreshMViews { - refreshMViews(conn) + if flagPostSnapshotImport { + err = importSchemaInternal(exportDir, []string{"TABLE"}, nil) + if err != nil { + return err + } + if flagRefreshMViews { + refreshMViews(conn) + } } else { utils.PrintAndLog("\nNOTE: Materialized Views are not populated by default. To populate them, pass --refresh-mviews while executing `import schema --post-snapshot-import`.") } diff --git a/yb-voyager/cmd/importSchemaYugabyteDB.go b/yb-voyager/cmd/importSchemaYugabyteDB.go index ec311172d7..e651d0a5f5 100644 --- a/yb-voyager/cmd/importSchemaYugabyteDB.go +++ b/yb-voyager/cmd/importSchemaYugabyteDB.go @@ -27,6 +27,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" + "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) @@ -49,6 +50,25 @@ func importSchemaInternal(exportDir string, importObjectList []string, return nil } +func isNotValidConstraint(stmt string) (bool, error) { + parseTree, err := queryparser.Parse(stmt) + if err != nil { + return false, fmt.Errorf("error parsing the ddl[%s]: %v", stmt, err) + } + ddlObj, err := queryparser.ProcessDDL(parseTree) + if err != nil { + return false, fmt.Errorf("error in process DDL[%s]:%v", stmt, err) + } + alter, ok := ddlObj.(*queryparser.AlterTable) + if !ok { + return false, nil + } + if alter.IsAddConstraintType() && alter.ConstraintNotValid { + return true, nil + } + return false, nil +} + func executeSqlFile(file string, objType string, skipFn func(string, string) bool) error { log.Infof("Execute SQL file %q on target %q", file, tconf.Host) conn := newTargetConn() @@ -74,10 +94,13 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo if objType == "TABLE" { stmt := strings.ToUpper(sqlInfo.stmt) - skip := strings.Contains(stmt, "ALTER TABLE") && strings.Contains(stmt, "REPLICA IDENTITY") + // Check if the statement should be skipped + skip, err := shouldSkipDDL(stmt) + if err != nil { + return fmt.Errorf("error checking whether to skip DDL: %v", err) + } if skip { - //skipping DDLS like ALTER TABLE ... REPLICA IDENTITY .. as this is not supported in YB - log.Infof("Skipping DDL: %s", sqlInfo.stmt) + log.Infof("Skipping DDL: %s", stmt) continue } } @@ -90,6 +113,24 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo return nil } +func shouldSkipDDL(stmt string) (bool, error) { + skipReplicaIdentity := strings.Contains(stmt, "ALTER TABLE") && strings.Contains(stmt, "REPLICA IDENTITY") + if skipReplicaIdentity { + return true, nil + } + isNotValid, err := isNotValidConstraint(stmt) + if err != nil { + return false, fmt.Errorf("error checking whether stmt is to add not valid constraint: %v", err) + } + skipNotValidWithoutPostImport := isNotValid && !bool(flagPostSnapshotImport) + skipOtherDDLsWithPostImport := (bool(flagPostSnapshotImport) && !isNotValid) + if skipNotValidWithoutPostImport || // Skipping NOT VALID CONSTRAINT in import schema without post-snapshot-mode + skipOtherDDLsWithPostImport { // Skipping other TABLE DDLs than the NOT VALID in post-snapshot-import mode + return true, nil + } + return false, nil +} + func executeSqlStmtWithRetries(conn **pgx.Conn, sqlInfo sqlInfo, objType string) error { var err error log.Infof("On %s run query:\n%s\n", tconf.Host, sqlInfo.formattedStmt) diff --git a/yb-voyager/src/queryparser/ddl_processor.go b/yb-voyager/src/queryparser/ddl_processor.go index c9598667bb..f7249d8896 100644 --- a/yb-voyager/src/queryparser/ddl_processor.go +++ b/yb-voyager/src/queryparser/ddl_processor.go @@ -555,11 +555,17 @@ func (atProcessor *AlterTableProcessor) Process(parseTree *pg_query.ParseResult) similar to CREATE table 2nd case where constraint is at the end of column definitions mentioning the constraint only so here as well while adding constraint checking the type of constraint and the deferrable field of it. + + ALTER TABLE test ADD CONSTRAINT chk check (id<>'') NOT VALID; + stmts:{stmt:...subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_CHECK conname:"chk" location:22 + raw_expr:{a_expr:{kind:AEXPR_OP name:{string:{sval:"<>"}} lexpr:{column_ref:{fields:{string:{sval:"id"}} location:43}} rexpr:{a_const:{sval:{} + location:47}} location:45}} skip_validation:true}} behavior:DROP_RESTRICT}} objtype:OBJECT_TABLE}} stmt_len:60} */ constraint := cmd.GetDef().GetConstraint() alter.ConstraintType = constraint.Contype alter.ConstraintName = constraint.Conname alter.IsDeferrable = constraint.Deferrable + alter.ConstraintNotValid = constraint.SkipValidation // this is set for the NOT VALID clause alter.ConstraintColumns = parseColumnsFromKeys(constraint.GetKeys()) case pg_query.AlterTableType_AT_DisableRule: @@ -590,10 +596,11 @@ type AlterTable struct { NumSetAttributes int NumStorageOptions int //In case AlterType - ADD_CONSTRAINT - ConstraintType pg_query.ConstrType - ConstraintName string - IsDeferrable bool - ConstraintColumns []string + ConstraintType pg_query.ConstrType + ConstraintName string + ConstraintNotValid bool + IsDeferrable bool + ConstraintColumns []string } func (a *AlterTable) GetObjectName() string { @@ -606,6 +613,10 @@ func (a *AlterTable) AddPrimaryKeyOrUniqueCons() bool { return a.ConstraintType == PRIMARY_CONSTR_TYPE || a.ConstraintType == UNIQUE_CONSTR_TYPE } +func (a *AlterTable) IsAddConstraintType() bool { + return a.AlterType == pg_query.AlterTableType_AT_AddConstraint +} + //===========POLICY PROCESSOR ================================ // PolicyProcessor handles parsing CREATE POLICY statements From e242809a31dd6b1e6e47f0bbd58141f4da4283fb Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:47:24 +0530 Subject: [PATCH 039/105] Adding unit tests to detect breaking changes in the code (#2010) --- yb-voyager/cmd/common_test.go | 429 ++++++++++++++++++ yb-voyager/cmd/exportDataStatus_test.go | 95 ++++ yb-voyager/go.mod | 93 +++- yb-voyager/go.sum | 189 ++++++-- yb-voyager/src/callhome/diagnostics_test.go | 204 +++++++++ yb-voyager/src/cp/yugabyted/yugabyted_test.go | 174 +++++++ yb-voyager/src/datafile/descriptor_test.go | 111 +++++ yb-voyager/src/dbzm/status_test.go | 50 ++ yb-voyager/src/metadb/metadataDB_test.go | 101 +++++ yb-voyager/src/migassessment/assessmentDB.go | 2 +- .../src/migassessment/assessmentDB_test.go | 131 ++++++ yb-voyager/src/namereg/namereg_test.go | 150 +++++- yb-voyager/src/testutils/testing_utils.go | 352 ++++++++++++++ yb-voyager/src/tgtdb/postgres_test.go | 82 ++++ yb-voyager/src/tgtdb/yugabytedb_test.go | 82 ++++ yb-voyager/testcontainers/testcontainers.go | 108 +++++ 16 files changed, 2279 insertions(+), 74 deletions(-) create mode 100644 yb-voyager/cmd/common_test.go create mode 100644 yb-voyager/cmd/exportDataStatus_test.go create mode 100644 yb-voyager/src/callhome/diagnostics_test.go create mode 100644 yb-voyager/src/cp/yugabyted/yugabyted_test.go create mode 100644 yb-voyager/src/datafile/descriptor_test.go create mode 100644 yb-voyager/src/dbzm/status_test.go create mode 100644 yb-voyager/src/metadb/metadataDB_test.go create mode 100644 yb-voyager/src/migassessment/assessmentDB_test.go create mode 100644 yb-voyager/src/testutils/testing_utils.go create mode 100644 yb-voyager/src/tgtdb/postgres_test.go create mode 100644 yb-voyager/src/tgtdb/yugabytedb_test.go create mode 100644 yb-voyager/testcontainers/testcontainers.go diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go new file mode 100644 index 0000000000..d0046ea7ad --- /dev/null +++ b/yb-voyager/cmd/common_test.go @@ -0,0 +1,429 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" +) + +func TestAssessmentReportStructs(t *testing.T) { + tests := []struct { + name string + actualType reflect.Type + expectedType interface{} + }{ + { + name: "Validate DBObject Struct Definition", + actualType: reflect.TypeOf(utils.DBObject{}), + expectedType: struct { + ObjectType string `json:"ObjectType"` + TotalCount int `json:"TotalCount"` + InvalidCount int `json:"InvalidCount"` + ObjectNames string `json:"ObjectNames"` + Details string `json:"Details,omitempty"` + }{}, + }, + { + name: "Validate SchemaSummary Struct Definition", + actualType: reflect.TypeOf(utils.SchemaSummary{}), + expectedType: struct { + Description string `json:"Description"` + DBName string `json:"DbName"` + SchemaNames []string `json:"SchemaNames"` + DBVersion string `json:"DbVersion"` + Notes []string `json:"Notes,omitempty"` + DBObjects []utils.DBObject `json:"DatabaseObjects"` + }{}, + }, + { + name: "Validate SizingRecommendation Struct Definition", + actualType: reflect.TypeOf(migassessment.SizingRecommendation{}), + expectedType: struct { + ColocatedTables []string + ColocatedReasoning string + ShardedTables []string + NumNodes float64 + VCPUsPerInstance int + MemoryPerInstance int + OptimalSelectConnectionsPerNode int64 + OptimalInsertConnectionsPerNode int64 + EstimatedTimeInMinForImport float64 + ParallelVoyagerJobs float64 + }{}, + }, + { + name: "Validate TableColumnsDataTypes Struct Definition", + actualType: reflect.TypeOf(utils.TableColumnsDataTypes{}), + expectedType: struct { + SchemaName string `json:"SchemaName"` + TableName string `json:"TableName"` + ColumnName string `json:"ColumnName"` + DataType string `json:"DataType"` + }{}, + }, + { + name: "Validate UnsupportedFeature Struct Definition", + actualType: reflect.TypeOf(UnsupportedFeature{}), + expectedType: struct { + FeatureName string `json:"FeatureName"` + Objects []ObjectInfo `json:"Objects"` + DisplayDDL bool `json:"-"` + DocsLink string `json:"DocsLink,omitempty"` + FeatureDescription string `json:"FeatureDescription,omitempty"` + MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"` + }{}, + }, + { + name: "Validate UnsupportedQueryConstruct Struct Definition", + actualType: reflect.TypeOf(utils.UnsupportedQueryConstruct{}), + expectedType: struct { + ConstructTypeName string + Query string + DocsLink string + MinimumVersionsFixedIn map[string]*ybversion.YBVersion + }{}, + }, + { + name: "Validate TableIndexStats Struct Definition", + actualType: reflect.TypeOf(migassessment.TableIndexStats{}), + expectedType: struct { + SchemaName string `json:"SchemaName"` + ObjectName string `json:"ObjectName"` + RowCount *int64 `json:"RowCount"` // Pointer to allows null values + ColumnCount *int64 `json:"ColumnCount"` + Reads *int64 `json:"Reads"` + Writes *int64 `json:"Writes"` + ReadsPerSecond *int64 `json:"ReadsPerSecond"` + WritesPerSecond *int64 `json:"WritesPerSecond"` + IsIndex bool `json:"IsIndex"` + ObjectType string `json:"ObjectType"` + ParentTableName *string `json:"ParentTableName"` + SizeInBytes *int64 `json:"SizeInBytes"` + }{}, + }, + { + name: "Validate AssessmentReport Struct Definition", + actualType: reflect.TypeOf(AssessmentReport{}), + expectedType: struct { + VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` + MigrationComplexity string `json:"MigrationComplexity"` + SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` + Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` + UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` + UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` + UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` + UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` + UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` + UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` + MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` + TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` + Notes []string `json:"Notes"` + }{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testutils.CompareStructs(t, tt.actualType, reflect.TypeOf(tt.expectedType), tt.name) + }) + } +} + +func TestAssessmentReportJson(t *testing.T) { + reportDir := filepath.Join(os.TempDir(), "assessment_report_test") + reportPath := filepath.Join(reportDir, fmt.Sprintf("%s%s", ASSESSMENT_FILE_NAME, JSON_EXTENSION)) + + newYbVersion, err := ybversion.NewYBVersion("2024.1.1.1") + if err != nil { + t.Fatalf("Failed to create new YBVersion: %v", err) + } + + assessmentReport = AssessmentReport{ + VoyagerVersion: "v1.0.0", + TargetDBVersion: newYbVersion, + MigrationComplexity: "High", + SchemaSummary: utils.SchemaSummary{ + Description: "Test Schema Summary", + DBName: "test_db", + SchemaNames: []string{"public"}, + DBVersion: "13.3", + DBObjects: []utils.DBObject{ + { + ObjectType: "Table", + TotalCount: 1, + InvalidCount: 0, + ObjectNames: "test_table", + }, + }, + }, + Sizing: &migassessment.SizingAssessmentReport{ + SizingRecommendation: migassessment.SizingRecommendation{ + ColocatedTables: []string{"test_table"}, + ColocatedReasoning: "Test reasoning", + ShardedTables: []string{"test_table"}, + NumNodes: 3, + VCPUsPerInstance: 4, + MemoryPerInstance: 16, + OptimalSelectConnectionsPerNode: 10, + OptimalInsertConnectionsPerNode: 10, + EstimatedTimeInMinForImport: 10, + ParallelVoyagerJobs: 10, + }, + FailureReasoning: "Test failure reasoning", + }, + UnsupportedDataTypes: []utils.TableColumnsDataTypes{ + { + SchemaName: "public", + TableName: "test_table", + ColumnName: "test_column", + DataType: "test_type", + }, + }, + UnsupportedDataTypesDesc: "Test unsupported data types", + UnsupportedFeatures: []UnsupportedFeature{ + { + FeatureName: "test_feature", + Objects: []ObjectInfo{ + { + ObjectName: "test_object", + ObjectType: "test_type", + SqlStatement: "test_sql", + }, + }, + DisplayDDL: true, + DocsLink: "https://test.com", + FeatureDescription: "Test feature description", + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{"2024.1.1": newYbVersion}, + }, + }, + UnsupportedFeaturesDesc: "Test unsupported features", + UnsupportedQueryConstructs: []utils.UnsupportedQueryConstruct{ + { + ConstructTypeName: "test_construct", + Query: "test_query", + DocsLink: "https://test.com", + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{"2024.1.1": newYbVersion}, + }, + }, + UnsupportedPlPgSqlObjects: []UnsupportedFeature{ + { + FeatureName: "test_feature", + Objects: []ObjectInfo{ + { + ObjectName: "test_object", + ObjectType: "test_type", + SqlStatement: "test_sql", + }, + }, + DisplayDDL: true, + DocsLink: "https://test.com", + FeatureDescription: "Test feature description", + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{"2024.1.1": newYbVersion}, + }, + }, + MigrationCaveats: []UnsupportedFeature{ + { + FeatureName: "test_feature", + Objects: []ObjectInfo{ + { + ObjectName: "test_object", + ObjectType: "test_type", + SqlStatement: "test_sql", + }, + }, + DisplayDDL: true, + DocsLink: "https://test.com", + FeatureDescription: "Test feature description", + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{"2024.1.1": newYbVersion}, + }, + }, + TableIndexStats: &[]migassessment.TableIndexStats{ + { + SchemaName: "public", + ObjectName: "test_table", + RowCount: Int64Ptr(100), + ColumnCount: Int64Ptr(10), + Reads: Int64Ptr(100), + Writes: Int64Ptr(100), + ReadsPerSecond: Int64Ptr(10), + WritesPerSecond: Int64Ptr(10), + IsIndex: true, + ObjectType: "Table", + ParentTableName: StringPtr("parent_table"), + SizeInBytes: Int64Ptr(1024), + }, + }, + Notes: []string{"Test note"}, + } + + // Make the report directory + err = os.MkdirAll(reportDir, 0755) + if err != nil { + t.Fatalf("Failed to create report directory: %v", err) + } + + // Clean up the report directory + defer func() { + err := os.RemoveAll(reportDir) + if err != nil { + t.Fatalf("Failed to remove report directory: %v", err) + } + }() + + // Write the assessment report to a JSON file + err = generateAssessmentReportJson(reportDir) + if err != nil { + t.Fatalf("Failed to write assessment report to JSON file: %v", err) + } + + // expected JSON + expectedJSON := `{ + "VoyagerVersion": "v1.0.0", + "TargetDBVersion": "2024.1.1.1", + "MigrationComplexity": "High", + "SchemaSummary": { + "Description": "Test Schema Summary", + "DbName": "test_db", + "SchemaNames": [ + "public" + ], + "DbVersion": "13.3", + "DatabaseObjects": [ + { + "ObjectType": "Table", + "TotalCount": 1, + "InvalidCount": 0, + "ObjectNames": "test_table" + } + ] + }, + "Sizing": { + "SizingRecommendation": { + "ColocatedTables": [ + "test_table" + ], + "ColocatedReasoning": "Test reasoning", + "ShardedTables": [ + "test_table" + ], + "NumNodes": 3, + "VCPUsPerInstance": 4, + "MemoryPerInstance": 16, + "OptimalSelectConnectionsPerNode": 10, + "OptimalInsertConnectionsPerNode": 10, + "EstimatedTimeInMinForImport": 10, + "ParallelVoyagerJobs": 10 + }, + "FailureReasoning": "Test failure reasoning" + }, + "UnsupportedDataTypes": [ + { + "SchemaName": "public", + "TableName": "test_table", + "ColumnName": "test_column", + "DataType": "test_type" + } + ], + "UnsupportedDataTypesDesc": "Test unsupported data types", + "UnsupportedFeatures": [ + { + "FeatureName": "test_feature", + "Objects": [ + { + "ObjectType": "test_type", + "ObjectName": "test_object", + "SqlStatement": "test_sql" + } + ], + "DocsLink": "https://test.com", + "FeatureDescription": "Test feature description", + "MinimumVersionsFixedIn": { + "2024.1.1": "2024.1.1.1" + } + } + ], + "UnsupportedFeaturesDesc": "Test unsupported features", + "UnsupportedQueryConstructs": [ + { + "ConstructTypeName": "test_construct", + "Query": "test_query", + "DocsLink": "https://test.com", + "MinimumVersionsFixedIn": { + "2024.1.1": "2024.1.1.1" + } + } + ], + "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "test_feature", + "Objects": [ + { + "ObjectType": "test_type", + "ObjectName": "test_object", + "SqlStatement": "test_sql" + } + ], + "DocsLink": "https://test.com", + "FeatureDescription": "Test feature description", + "MinimumVersionsFixedIn": { + "2024.1.1": "2024.1.1.1" + } + } + ], + "MigrationCaveats": [ + { + "FeatureName": "test_feature", + "Objects": [ + { + "ObjectType": "test_type", + "ObjectName": "test_object", + "SqlStatement": "test_sql" + } + ], + "DocsLink": "https://test.com", + "FeatureDescription": "Test feature description", + "MinimumVersionsFixedIn": { + "2024.1.1": "2024.1.1.1" + } + } + ], + "TableIndexStats": [ + { + "SchemaName": "public", + "ObjectName": "test_table", + "RowCount": 100, + "ColumnCount": 10, + "Reads": 100, + "Writes": 100, + "ReadsPerSecond": 10, + "WritesPerSecond": 10, + "IsIndex": true, + "ObjectType": "Table", + "ParentTableName": "parent_table", + "SizeInBytes": 1024 + } + ], + "Notes": [ + "Test note" + ] +}` + + testutils.CompareJson(t, reportPath, expectedJSON, reportDir) + +} + +func Int64Ptr(i int64) *int64 { + return &i +} + +func StringPtr(s string) *string { + return &s +} diff --git a/yb-voyager/cmd/exportDataStatus_test.go b/yb-voyager/cmd/exportDataStatus_test.go new file mode 100644 index 0000000000..beec85005c --- /dev/null +++ b/yb-voyager/cmd/exportDataStatus_test.go @@ -0,0 +1,95 @@ +package cmd + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" +) + +func TestExportSnapshotStatusStructs(t *testing.T) { + + test := []struct { + name string + actualType reflect.Type + expectedType interface{} + }{ + { + name: "Validate TableExportStatus Struct Definition", + actualType: reflect.TypeOf(TableExportStatus{}), + expectedType: struct { + TableName string `json:"table_name"` + FileName string `json:"file_name"` + Status string `json:"status"` + ExportedRowCountSnapshot int64 `json:"exported_row_count_snapshot"` + }{}, + }, + { + name: "Validate ExportSnapshotStatus Struct Definition", + actualType: reflect.TypeOf(ExportSnapshotStatus{}), + expectedType: struct { + Tables map[string]*TableExportStatus `json:"tables"` + }{}, + }, + } + + for _, tt := range test { + t.Run(tt.name, func(t *testing.T) { + testutils.CompareStructs(t, tt.actualType, reflect.TypeOf(tt.expectedType), tt.name) + }) + } +} + +func TestExportSnapshotStatusJson(t *testing.T) { + // Create a table list of type []sqlname.NameTuple + o1 := sqlname.NewObjectName(POSTGRESQL, "public", "public", "table1") + o2 := sqlname.NewObjectName(POSTGRESQL, "public", "schema1", "table2") + tableList := []sqlname.NameTuple{ + {CurrentName: o1, SourceName: o1, TargetName: o1}, + {CurrentName: o2, SourceName: o2, TargetName: o2}, + } + + exportDir = filepath.Join(os.TempDir(), "export_snapshot_status_test") + + // Make export directory + err := os.MkdirAll(filepath.Join(exportDir, "metainfo"), 0755) + if err != nil { + t.Fatalf("failed to create export directory: %v", err) + } + + // Clean up the export directory + defer func() { + err := os.RemoveAll(exportDir) + if err != nil { + t.Fatalf("failed to remove export directory: %v", err) + } + }() + + outputFilePath := filepath.Join(exportDir, "metainfo", "export_snapshot_status.json") + + // Call initializeExportTableMetadata to create the export_snapshot_status.json file + initializeExportTableMetadata(tableList) + + expectedExportSnapshotStatusJSON := `{ + "tables": { + "public.\"table1\"": { + "table_name": "public.\"table1\"", + "file_name": "", + "status": "NOT-STARTED", + "exported_row_count_snapshot": 0 + }, + "schema1.\"table2\"": { + "table_name": "schema1.\"table2\"", + "file_name": "", + "status": "NOT-STARTED", + "exported_row_count_snapshot": 0 + } + } +}` + + // Compare the JSON representation of the sample ExportSnapshotStatus instance + testutils.CompareJson(t, outputFilePath, expectedExportSnapshotStatusJSON, exportDir) +} diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 8f4c858131..b761c2d4a7 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -3,7 +3,7 @@ module github.com/yugabyte/yb-voyager/yb-voyager go 1.23.1 require ( - cloud.google.com/go/storage v1.29.0 + cloud.google.com/go/storage v1.38.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/DATA-DOG/go-sqlmock v1.5.2 @@ -13,11 +13,13 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 + github.com/fergusstrange/embedded-postgres v1.29.0 github.com/go-sql-driver/mysql v1.7.0 github.com/godror/godror v0.30.2 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gosuri/uilive v0.0.4 github.com/gosuri/uitable v0.0.4 + github.com/hashicorp/go-version v1.7.0 github.com/jackc/pgconn v1.13.0 github.com/jackc/pgx/v4 v4.17.2 github.com/jackc/pgx/v5 v5.0.3 @@ -27,35 +29,76 @@ require ( github.com/nightlyone/lockfile v1.0.0 github.com/pganalyze/pg_query_go/v5 v5.1.0 github.com/samber/lo v1.38.1 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.13.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/tebeka/atexit v0.3.0 + github.com/testcontainers/testcontainers-go v0.34.0 github.com/vbauerster/mpb/v8 v8.4.0 gocloud.dev v0.29.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/term v0.24.0 - google.golang.org/api v0.118.0 + google.golang.org/api v0.169.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( - github.com/fergusstrange/embedded-postgres v1.29.0 // indirect - github.com/hashicorp/go-version v1.7.0 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/lib/pq v1.10.9 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect ) require ( - cloud.google.com/go v0.110.2 // indirect - cloud.google.com/go/compute v1.19.0 // indirect + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/compute v1.25.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/iam v1.1.6 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -86,12 +129,12 @@ require ( github.com/godror/knownpb v0.1.0 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/s2a-go v0.1.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.6.0 + github.com/google/s2a-go v0.1.7 // indirect github.com/google/wire v0.5.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.8.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -102,7 +145,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.12.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/magiconair/properties v1.8.6 + github.com/magiconair/properties v1.8.7 github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -120,17 +163,17 @@ require ( github.com/spf13/pflag v1.0.5 github.com/subosito/gotenv v1.4.1 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.55.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/grpc v1.64.1 // indirect + google.golang.org/protobuf v1.33.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 58e148305e..a0de897915 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -39,8 +39,8 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.109.0/go.mod h1:2sYycXt75t/CSB5R9M2wPU1tJmire7AQZTPtITcGBVE= -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= @@ -124,8 +124,8 @@ cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARy cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= -cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= -cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -218,8 +218,8 @@ cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHD cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= @@ -240,8 +240,6 @@ cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9 cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.0/go.mod h1:eF3Qsw58iX/bkKtVjMTYpH0LRjQ2goDkjkNQTlzq/ZM= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= @@ -369,8 +367,9 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= +cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= @@ -417,8 +416,12 @@ code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/stackdriver v0.13.14/go.mod h1:5pSSGY0Bhuk7waTHuDf4aQ8D2DrhgETRo9fy6k3Xlzc= contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-amqp-common-go/v3 v3.2.3/go.mod h1:7rPmbSfszeovxGfc5fSAXE4ehlXQZHpMja2OtxC2Tas= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v63.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -448,6 +451,7 @@ github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fw github.com/Azure/go-amqp v0.18.1/go.mod h1:+bg0x3ce5+Q3ahCEXnCsGG3ETpDQe3MEVnOuT2ywPwc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -494,6 +498,8 @@ github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JP github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= @@ -641,6 +647,8 @@ github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRt github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -725,6 +733,8 @@ github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTV github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -753,9 +763,13 @@ github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6T github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= @@ -803,12 +817,16 @@ github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= @@ -827,6 +845,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.78.0/go.mod h1:GBmu8MkjZmNARE7IXRPmkbbnocNN8+uBm0xbEVw2LCs= github.com/digitalocean/godo v1.95.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= @@ -839,13 +859,18 @@ github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.23+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -890,6 +915,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fergusstrange/embedded-postgres v1.29.0 h1:Uv8hdhoiaNMuH0w8UuGXDHr60VoAQPFdgx7Qf3bzXJM= github.com/fergusstrange/embedded-postgres v1.29.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -938,8 +965,13 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= @@ -1043,6 +1075,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= @@ -1089,8 +1122,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -1154,23 +1187,25 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20220318212150-b2ab0324ddda/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.0 h1:3Qm0liEiCErViKERO2Su5wp+9PfMRiuS6XB5FvpKnYQ= -github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -1181,8 +1216,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= -github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -1217,10 +1252,13 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgvt8MNctRQHjEH8ubKtt1Ka3w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hanwen/go-fuse/v2 v2.2.0/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -1405,6 +1443,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1432,19 +1472,21 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linode/linodego v1.4.0/go.mod h1:PVsRxSlOiJyvG4/scTszpmZDTdgS+to3X6eS8pRrWI8= github.com/linode/linodego v1.12.0/go.mod h1:NJlzvlNtdMRRkXb0oN6UWzUkj6t+IBsyveHgZ5Ppjyk= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1522,17 +1564,27 @@ github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -1541,6 +1593,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -1608,11 +1661,14 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -1669,6 +1725,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/alertmanager v0.24.0/go.mod h1:r6fy/D7FRuZh5YbnX6J3MBY0eI4Pb5yPYS7/bPSXXqI= @@ -1761,7 +1819,13 @@ github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.0/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -1777,8 +1841,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -1829,6 +1893,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1840,8 +1906,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -1852,7 +1919,13 @@ github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ github.com/tebeka/atexit v0.3.0 h1:jleL99H7Ywt80oJKR+VWmJNnezcCOG0CuzcN3CIpsdI= github.com/tebeka/atexit v0.3.0/go.mod h1:WJmSUSmMT7WoR7etUOaGBVXk+f5/ZJ+67qwuedq7Fbs= github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= +github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -1899,6 +1972,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= @@ -1938,15 +2013,22 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0/go.mod h1:PFmBsWbldL1kiWZk9+0LBZz2brhByaGsvp6pRICMlPE= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= go.opentelemetry.io/otel v1.6.0/go.mod h1:bfJD2DZVw0LBxghOTlgnlI0CV3hLDu9XF/QKOUXMTQQ= go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ= go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.1/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40= @@ -1954,21 +2036,29 @@ go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2/go.mod h1:rqbht/L go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.1/go.mod h1:YJ/JbY5ag/tSQFXzH3mtDmHqzF3aFn3DI/aB1n7pt4w= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2/go.mod h1:5Qn6qvgkMsLDX+sYK64rHb1FPhpn0UtxF+ouX1uhyJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.1/go.mod h1:UJJXJj0rltNIemDMwkOJyggsvyMG9QHfJeFH0HS5JjM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2/go.mod h1:jWZUM2MWhWCJ9J9xVbRx7tzK1mXKpAlze4CeulycwVY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.6.1/go.mod h1:DAKwdo06hFLc0U88O10x4xnb5sc7dDRDqRuiN+io8JE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2/go.mod h1:GZWSQQky8AgdJj50r1KJm8oiQiIPaAX7uZCFQX9GzC8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v0.28.0/go.mod h1:TrzsfQAmQaB1PDcdhBauLMk7nyyg9hm+GoQq/ekE9Iw= go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= go.opentelemetry.io/otel/sdk v1.6.1/go.mod h1:IVYrddmFZ+eJqu2k38qD3WezFR2pymCzm8tdxyh3R4E= go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= @@ -1977,11 +2067,15 @@ go.opentelemetry.io/otel/trace v1.6.0/go.mod h1:qs7BrU5cZ8dXQHBGxHMOxwME/27YH2qE go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0= go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.opentelemetry.io/proto/otlp v0.12.1/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1995,6 +2089,7 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -2054,8 +2149,8 @@ golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2194,8 +2289,8 @@ golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmL golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2227,8 +2322,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2329,6 +2424,7 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2400,6 +2496,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2428,8 +2527,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2443,6 +2542,8 @@ golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2540,8 +2641,9 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= @@ -2607,16 +2709,17 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= -google.golang.org/api v0.118.0 h1:FNfHq9Z2GKULxu7cEhCaB0wWQHg43UpomrrN+24ZRdE= -google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= +google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -2753,8 +2856,12 @@ google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -2799,8 +2906,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.52.1/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2817,8 +2924,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2868,9 +2976,12 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go new file mode 100644 index 0000000000..2575a8e405 --- /dev/null +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -0,0 +1,204 @@ +package callhome + +import ( + "reflect" + "testing" + + "github.com/google/uuid" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" +) + +func TestCallhomeStructs(t *testing.T) { + + tests := []struct { + name string + actualType reflect.Type + expectedType interface{} + }{ + { + name: "Validate Payload Struct Definition", + actualType: reflect.TypeOf(Payload{}), + expectedType: struct { + MigrationUUID uuid.UUID `json:"migration_uuid"` + PhaseStartTime string `json:"phase_start_time"` + CollectedAt string `json:"collected_at"` + SourceDBDetails string `json:"source_db_details"` + TargetDBDetails string `json:"target_db_details"` + YBVoyagerVersion string `json:"yb_voyager_version"` + MigrationPhase string `json:"migration_phase"` + PhasePayload string `json:"phase_payload"` + MigrationType string `json:"migration_type"` + TimeTakenSec int `json:"time_taken_sec"` + Status string `json:"status"` + }{}, + }, + { + name: "Validate SourceDBDetails Struct Definition", + actualType: reflect.TypeOf(SourceDBDetails{}), + expectedType: struct { + Host string `json:"host"` + DBType string `json:"db_type"` + DBVersion string `json:"db_version"` + DBSize int64 `json:"total_db_size_bytes"` + Role string `json:"role,omitempty"` + }{}, + }, + { + name: "Validate TargetDBDetails Struct Definition", + actualType: reflect.TypeOf(TargetDBDetails{}), + expectedType: struct { + Host string `json:"host"` + DBVersion string `json:"db_version"` + NodeCount int `json:"node_count"` + Cores int `json:"total_cores"` + }{}, + }, + { + name: "Validate UnsupportedFeature Struct Definition", + actualType: reflect.TypeOf(UnsupportedFeature{}), + expectedType: struct { + FeatureName string `json:"FeatureName"` + Objects []string `json:"Objects,omitempty"` + ObjectCount int `json:"ObjectCount"` + TotalOccurrences int `json:"TotalOccurrences"` + }{}, + }, + { + name: "Validate AssessMigrationPhasePayload Struct Definition", + actualType: reflect.TypeOf(AssessMigrationPhasePayload{}), + expectedType: struct { + MigrationComplexity string `json:"migration_complexity"` + UnsupportedFeatures string `json:"unsupported_features"` + UnsupportedDatatypes string `json:"unsupported_datatypes"` + UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` + MigrationCaveats string `json:"migration_caveats"` + UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` + Error string `json:"error,omitempty"` + TableSizingStats string `json:"table_sizing_stats"` + IndexSizingStats string `json:"index_sizing_stats"` + SchemaSummary string `json:"schema_summary"` + SourceConnectivity bool `json:"source_connectivity"` + IopsInterval int64 `json:"iops_interval"` + }{}, + }, + { + name: "Validate AssessMigrationBulkPhasePayload Struct Definition", + actualType: reflect.TypeOf(AssessMigrationBulkPhasePayload{}), + expectedType: struct { + FleetConfigCount int `json:"fleet_config_count"` + }{}, + }, + { + name: "Validate ObjectSizingStats Struct Definition", + actualType: reflect.TypeOf(ObjectSizingStats{}), + expectedType: struct { + SchemaName string `json:"schema_name,omitempty"` + ObjectName string `json:"object_name"` + ReadsPerSecond int64 `json:"reads_per_second"` + WritesPerSecond int64 `json:"writes_per_second"` + SizeInBytes int64 `json:"size_in_bytes"` + }{}, + }, + { + name: "Validate ExportSchemaPhasePayload Struct Definition", + actualType: reflect.TypeOf(ExportSchemaPhasePayload{}), + expectedType: struct { + StartClean bool `json:"start_clean"` + AppliedRecommendations bool `json:"applied_recommendations"` + UseOrafce bool `json:"use_orafce"` + CommentsOnObjects bool `json:"comments_on_objects"` + }{}, + }, + { + name: "Validate AnalyzePhasePayload Struct Definition", + actualType: reflect.TypeOf(AnalyzePhasePayload{}), + expectedType: struct { + Issues string `json:"issues"` + DatabaseObjects string `json:"database_objects"` + }{}, + }, + { + name: "Validate ExportDataPhasePayload Struct Definition", + actualType: reflect.TypeOf(ExportDataPhasePayload{}), + expectedType: struct { + ParallelJobs int64 `json:"parallel_jobs"` + TotalRows int64 `json:"total_rows_exported"` + LargestTableRows int64 `json:"largest_table_rows_exported"` + StartClean bool `json:"start_clean"` + ExportSnapshotMechanism string `json:"export_snapshot_mechanism,omitempty"` + Phase string `json:"phase,omitempty"` + TotalExportedEvents int64 `json:"total_exported_events,omitempty"` + EventsExportRate int64 `json:"events_export_rate_3m,omitempty"` + LiveWorkflowType string `json:"live_workflow_type,omitempty"` + }{}, + }, + { + name: "Validate ImportSchemaPhasePayload Struct Definition", + actualType: reflect.TypeOf(ImportSchemaPhasePayload{}), + expectedType: struct { + ContinueOnError bool `json:"continue_on_error"` + EnableOrafce bool `json:"enable_orafce"` + IgnoreExist bool `json:"ignore_exist"` + RefreshMviews bool `json:"refresh_mviews"` + ErrorCount int `json:"errors"` + PostSnapshotImport bool `json:"post_snapshot_import"` + StartClean bool `json:"start_clean"` + }{}, + }, + { + name: "Validate ImportDataPhasePayload Struct Definition", + actualType: reflect.TypeOf(ImportDataPhasePayload{}), + expectedType: struct { + ParallelJobs int64 `json:"parallel_jobs"` + TotalRows int64 `json:"total_rows_imported"` + LargestTableRows int64 `json:"largest_table_rows_imported"` + StartClean bool `json:"start_clean"` + Phase string `json:"phase,omitempty"` + TotalImportedEvents int64 `json:"total_imported_events,omitempty"` + EventsImportRate int64 `json:"events_import_rate_3m,omitempty"` + LiveWorkflowType string `json:"live_workflow_type,omitempty"` + EnableUpsert bool `json:"enable_upsert"` + }{}, + }, + { + name: "Validate ImportDataFilePhasePayload Struct Definition", + actualType: reflect.TypeOf(ImportDataFilePhasePayload{}), + expectedType: struct { + ParallelJobs int64 `json:"parallel_jobs"` + TotalSize int64 `json:"total_size_imported"` + LargestTableSize int64 `json:"largest_table_size_imported"` + FileStorageType string `json:"file_storage_type"` + StartClean bool `json:"start_clean"` + DataFileParameters string `json:"data_file_parameters"` + }{}, + }, + { + name: "Validate DataFileParameters Struct Definition", + actualType: reflect.TypeOf(DataFileParameters{}), + expectedType: struct { + FileFormat string `json:"FileFormat"` + Delimiter string `json:"Delimiter"` + HasHeader bool `json:"HasHeader"` + QuoteChar string `json:"QuoteChar,omitempty"` + EscapeChar string `json:"EscapeChar,omitempty"` + NullString string `json:"NullString,omitempty"` + }{}, + }, + { + name: "Validate EndMigrationPhasePayload Struct Definition", + actualType: reflect.TypeOf(EndMigrationPhasePayload{}), + expectedType: struct { + BackupDataFiles bool `json:"backup_data_files"` + BackupLogFiles bool `json:"backup_log_files"` + BackupSchemaFiles bool `json:"backup_schema_files"` + SaveMigrationReports bool `json:"save_migration_reports"` + }{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testutils.CompareStructs(t, tt.actualType, reflect.TypeOf(tt.expectedType), tt.name) + }) + } +} diff --git a/yb-voyager/src/cp/yugabyted/yugabyted_test.go b/yb-voyager/src/cp/yugabyted/yugabyted_test.go new file mode 100644 index 0000000000..1c6690999d --- /dev/null +++ b/yb-voyager/src/cp/yugabyted/yugabyted_test.go @@ -0,0 +1,174 @@ +package yugabyted + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v4/pgxpool" + _ "github.com/lib/pq" // PostgreSQL driver + "github.com/stretchr/testify/assert" + controlPlane "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" +) + +func TestYugabyteDTableSchema(t *testing.T) { + ctx := context.Background() + + // Start a YugabyteDB container + ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) + assert.NoError(t, err, "Failed to start YugabyteDB container") + defer ybContainer.Terminate(ctx) + + // Connect to the database + dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) + db, err := sql.Open("postgres", dsn) + assert.NoError(t, err) + defer db.Close() + + // Wait for the database to be ready + err = testcontainers.WaitForDBToBeReady(db) + assert.NoError(t, err) + // Export the database connection string to env variable YUGABYTED_DB_CONN_STRING + err = os.Setenv("YUGABYTED_DB_CONN_STRING", dsn) + + exportDir := filepath.Join(os.TempDir(), "yugabyted") + + // Create a temporary export directory for testing + err = os.MkdirAll(exportDir, 0755) + assert.NoError(t, err, "Failed to create temporary export directory") + // Ensure the directory is removed after the test + defer func() { + err := os.RemoveAll(exportDir) + assert.NoError(t, err, "Failed to remove temporary export directory") + }() + + controlPlane := New(exportDir) + controlPlane.eventChan = make(chan MigrationEvent, 100) + controlPlane.rowCountUpdateEventChan = make(chan []VisualizerTableMetrics, 200) + + err = controlPlane.connect() + assert.NoError(t, err, "Failed to connect to YugabyteDB") + + err = controlPlane.setupDatabase() + assert.NoError(t, err, "Failed to setup YugabyteDB database") + + expectedTables := map[string]map[string]testutils.ColumnPropertiesPG{ + QUALIFIED_YUGABYTED_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "migration_phase": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "invocation_sequence": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "migration_dir": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "database_name": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "schema_name": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "payload": {Type: "text", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "complexity": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "db_type": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "status": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "invocation_timestamp": {Type: "timestamp with time zone", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "host_ip": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "port": {Type: "integer", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "db_version": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "voyager_info": {Type: "character varying", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + YUGABYTED_TABLE_METRICS_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "schema_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "migration_phase": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "status": {Type: "integer", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "count_live_rows": {Type: "integer", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "count_total_rows": {Type: "integer", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "invocation_timestamp": {Type: "timestamp with time zone", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + } + + // Validate the schema and tables + t.Run("Check all the expected tables and no extra tables", func(t *testing.T) { + testutils.CheckTableExistencePG(t, db, VISUALIZER_METADATA_SCHEMA, expectedTables) + }) + + // Validate columns for each table + for tableName, expectedColumns := range expectedTables { + t.Run(fmt.Sprintf("Check columns for %s table", tableName), func(t *testing.T) { + table := strings.Split(tableName, ".")[1] + testutils.CheckTableStructurePG(t, db, VISUALIZER_METADATA_SCHEMA, table, expectedColumns) + }) + } +} + +func TestYugabyteDStructs(t *testing.T) { + // Test the structs used in YugabyteD + + expectedVoyagerInstance := struct { + IP string + OperatingSystem string + DiskSpaceAvailable uint64 + ExportDirectory string + }{} + + t.Run("Validate VoyagerInstance Struct Definition", func(t *testing.T) { + testutils.CompareStructs(t, reflect.TypeOf(controlPlane.VoyagerInstance{}), reflect.TypeOf(expectedVoyagerInstance), "VoyagerInstance") + }) + + expectedMigrationEvent := struct { + MigrationUUID uuid.UUID `json:"migration_uuid"` + MigrationPhase int `json:"migration_phase"` + InvocationSequence int `json:"invocation_sequence"` + MigrationDirectory string `json:"migration_dir"` + DatabaseName string `json:"database_name"` + SchemaName string `json:"schema_name"` + DBIP string `json:"db_ip"` + Port int `json:"port"` + DBVersion string `json:"db_version"` + Payload string `json:"payload"` + VoyagerInfo string `json:"voyager_info"` + DBType string `json:"db_type"` + Status string `json:"status"` + InvocationTimestamp string `json:"invocation_timestamp"` + }{} + + t.Run("Validate MigrationEvent Struct Definition", func(t *testing.T) { + testutils.CompareStructs(t, reflect.TypeOf(MigrationEvent{}), reflect.TypeOf(expectedMigrationEvent), "MigrationEvent") + }) + + expectedVisualizerTableMetrics := struct { + MigrationUUID uuid.UUID `json:"migration_uuid"` + TableName string `json:"table_name"` + Schema string `json:"schema_name"` + MigrationPhase int `json:"migration_phase"` + Status int `json:"status"` + CountLiveRows int64 `json:"count_live_rows"` + CountTotalRows int64 `json:"count_total_rows"` + InvocationTimestamp string `json:"invocation_timestamp"` + }{} + + t.Run("Validate VisualizerTableMetrics Struct Definition", func(t *testing.T) { + testutils.CompareStructs(t, reflect.TypeOf(VisualizerTableMetrics{}), reflect.TypeOf(expectedVisualizerTableMetrics), "VisualizerTableMetrics") + }) + + expectedYugabyteD := struct { + sync.Mutex + migrationDirectory string + voyagerInfo *controlPlane.VoyagerInstance + waitGroup sync.WaitGroup + eventChan chan (MigrationEvent) + rowCountUpdateEventChan chan ([]VisualizerTableMetrics) + connPool *pgxpool.Pool + lastRowCountUpdate map[string]time.Time + latestInvocationSequence int + }{} + + t.Run("Validate YugabyteD Struct Definition", func(t *testing.T) { + testutils.CompareStructs(t, reflect.TypeOf(&YugabyteD{}).Elem(), reflect.TypeOf(&expectedYugabyteD).Elem(), "YugabyteD") + }) +} diff --git a/yb-voyager/src/datafile/descriptor_test.go b/yb-voyager/src/datafile/descriptor_test.go new file mode 100644 index 0000000000..dc617d0909 --- /dev/null +++ b/yb-voyager/src/datafile/descriptor_test.go @@ -0,0 +1,111 @@ +package datafile + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" +) + +func TestDescriptorStructs(t *testing.T) { + // Define the expected structure for FileEntry + expectedFileEntry := struct { + FilePath string `json:"FilePath"` + TableName string `json:"TableName"` + RowCount int64 `json:"RowCount"` + FileSize int64 `json:"FileSize"` + }{} + + // Define the expected structure for Descriptor + expectedDescriptor := struct { + FileFormat string `json:"FileFormat"` + Delimiter string `json:"Delimiter"` + HasHeader bool `json:"HasHeader"` + ExportDir string `json:"-"` + QuoteChar byte `json:"QuoteChar,omitempty"` + EscapeChar byte `json:"EscapeChar,omitempty"` + NullString string `json:"NullString,omitempty"` + DataFileList []*FileEntry `json:"FileList"` + TableNameToExportedColumns map[string][]string `json:"TableNameToExportedColumns"` + }{} + + t.Run("Validate FileEntry Struct Definition", func(t *testing.T) { + testutils.CompareStructs(t, reflect.TypeOf(FileEntry{}), reflect.TypeOf(expectedFileEntry), "FileEntry") + }) + + t.Run("Validate Descriptor Struct Definition", func(t *testing.T) { + testutils.CompareStructs(t, reflect.TypeOf(Descriptor{}), reflect.TypeOf(expectedDescriptor), "Descriptor") + }) +} + +func TestDescriptorJson(t *testing.T) { + // Set up the temporary export directory + exportDir := filepath.Join(os.TempDir(), "descriptor_test") + outputFilePath := filepath.Join(exportDir, DESCRIPTOR_PATH) + + // Create a sample Descriptor instance + descriptor := Descriptor{ + FileFormat: "csv", + Delimiter: ",", + HasHeader: true, + ExportDir: exportDir, + QuoteChar: '"', + EscapeChar: '\\', + NullString: "NULL", + DataFileList: []*FileEntry{ + { + FilePath: "file.csv", // Use relative path for testing absolute path handling. + TableName: "public.my_table", + RowCount: 100, + FileSize: 2048, + }, + }, + TableNameToExportedColumns: map[string][]string{ + "public.my_table": {"id", "name", "age"}, + }, + } + + // Ensure the export directory exists + if err := os.MkdirAll(filepath.Join(exportDir, "metainfo"), 0755); err != nil { + t.Fatalf("Failed to create export directory: %v", err) + } + + // Clean up the export directory + defer func() { + if err := os.RemoveAll(exportDir); err != nil { + t.Fatalf("Failed to remove export directory: %v", err) + } + }() + + // Save the Descriptor to JSON + descriptor.Save() + + expectedJSON := `{ + "FileFormat": "csv", + "Delimiter": ",", + "HasHeader": true, + "QuoteChar": 34, + "EscapeChar": 92, + "NullString": "NULL", + "FileList": [ + { + "FilePath": "file.csv", + "TableName": "public.my_table", + "RowCount": 100, + "FileSize": 2048 + } + ], + "TableNameToExportedColumns": { + "public.my_table": [ + "id", + "name", + "age" + ] + } +}` + + // Compare the output JSON with the expected JSON + testutils.CompareJson(t, outputFilePath, expectedJSON, exportDir) +} diff --git a/yb-voyager/src/dbzm/status_test.go b/yb-voyager/src/dbzm/status_test.go new file mode 100644 index 0000000000..9b384e9ad4 --- /dev/null +++ b/yb-voyager/src/dbzm/status_test.go @@ -0,0 +1,50 @@ +package dbzm + +import ( + "reflect" + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" +) + +func TestExportStatusStructs(t *testing.T) { + test := []struct { + name string + actualType reflect.Type + expectedType interface{} + }{ + { + name: "Validate TableExportStatus Struct Definition", + actualType: reflect.TypeOf(TableExportStatus{}), + expectedType: struct { + Sno int `json:"sno"` + DatabaseName string `json:"database_name"` + SchemaName string `json:"schema_name"` + TableName string `json:"table_name"` + FileName string `json:"file_name"` + ExportedRowCountSnapshot int64 `json:"exported_row_count_snapshot"` + }{}, + }, + { + name: "Validate ExportStatus Struct Definition", + actualType: reflect.TypeOf(ExportStatus{}), + expectedType: struct { + Mode string `json:"mode"` + Tables []TableExportStatus `json:"tables"` + Sequences map[string]int64 `json:"sequences"` + }{}, + }, + } + + for _, tt := range test { + t.Run(tt.name, func(t *testing.T) { + testutils.CompareStructs(t, tt.actualType, reflect.TypeOf(tt.expectedType), tt.name) + }) + } +} + +// TODO: Implement this test +// The export status json file is created by debezium and currently we dont have infrastructure to test it. +// To test this we need to create a json file (using dbzm code) and read it back (here) and compare the values. +// func TestReadExportStatus(t *testing.T) { +//} diff --git a/yb-voyager/src/metadb/metadataDB_test.go b/yb-voyager/src/metadb/metadataDB_test.go new file mode 100644 index 0000000000..89648a26b7 --- /dev/null +++ b/yb-voyager/src/metadb/metadataDB_test.go @@ -0,0 +1,101 @@ +package metadb + +import ( + "database/sql" + "fmt" + "os" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" +) + +// Test the initMetaDB function +func TestInitMetaDB(t *testing.T) { + // Define the expected columns and their types for each table + expectedTables := map[string]map[string]testutils.ColumnPropertiesSqlite{ + QUEUE_SEGMENT_META_TABLE_NAME: { + "segment_no": {Type: "INTEGER", PrimaryKey: 1}, + "file_path": {Type: "TEXT"}, + "size_committed": {Type: "INTEGER"}, + "total_events": {Type: "INTEGER"}, + "exporter_role": {Type: "TEXT"}, + "imported_by_target_db_importer": {Type: "INTEGER", Default: sql.NullString{String: "0", Valid: true}}, + "imported_by_source_replica_db_importer": {Type: "INTEGER", Default: sql.NullString{String: "0", Valid: true}}, + "imported_by_source_db_importer": {Type: "INTEGER", Default: sql.NullString{String: "0", Valid: true}}, + "archived": {Type: "INTEGER", Default: sql.NullString{String: "0", Valid: true}}, + "deleted": {Type: "INTEGER", Default: sql.NullString{String: "0", Valid: true}}, + "archive_location": {Type: "TEXT"}, + }, + EXPORTED_EVENTS_STATS_TABLE_NAME: { + // TODO: We have a composite primary key here (run_id, exporter_role, timestamp) + "run_id": {Type: "TEXT", PrimaryKey: 1}, + "exporter_role": {Type: "TEXT", PrimaryKey: 2}, + "timestamp": {Type: "INTEGER", PrimaryKey: 3}, + "num_total": {Type: "INTEGER"}, + "num_inserts": {Type: "INTEGER"}, + "num_updates": {Type: "INTEGER"}, + "num_deletes": {Type: "INTEGER"}, + }, + EXPORTED_EVENTS_STATS_PER_TABLE_TABLE_NAME: { + "exporter_role": {Type: "TEXT", PrimaryKey: 1}, + "schema_name": {Type: "TEXT", PrimaryKey: 2}, + "table_name": {Type: "TEXT", PrimaryKey: 3}, + "num_total": {Type: "INTEGER"}, + "num_inserts": {Type: "INTEGER"}, + "num_updates": {Type: "INTEGER"}, + "num_deletes": {Type: "INTEGER"}, + }, + JSON_OBJECTS_TABLE_NAME: { + "key": {Type: "TEXT", PrimaryKey: 1}, + "json_text": {Type: "TEXT"}, + }, + } + + // Create a temporary SQLite database file for testing + tempFile, err := os.CreateTemp(os.TempDir(), "test_meta_db_*.db") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + + // remove the temporary file + defer func() { + err := os.Remove(tempFile.Name()) + if err != nil { + t.Fatalf("Failed to remove temporary file: %v", err) + } + }() + + // Call initMetaDB with the path to the temporary file + err = initMetaDB(tempFile.Name()) // Pass the temp file path to initMetaDB + if err != nil { + t.Fatalf("Failed to initialize database: %v", err) + } else { + t.Logf("Database initialized successfully") + } + + // Open the temporary database for verification + db, err := sql.Open("sqlite3", tempFile.Name()) + if err != nil { + t.Fatalf("Failed to open temporary database: %v", err) + } + defer db.Close() + + // Verify the existence of each table and no extra tables + t.Run("Check table existence and no extra tables", func(t *testing.T) { + err := testutils.CheckTableExistenceSqlite(t, db, expectedTables) + if err != nil { + t.Errorf("Table existence mismatch: %v", err) + } + }) + + // Verify the structure of each table + for table, expectedColumns := range expectedTables { + t.Run(fmt.Sprintf("Check structure of %s table", table), func(t *testing.T) { + err := testutils.CheckTableStructureSqlite(db, table, expectedColumns) + if err != nil { + t.Errorf("Table %s structure mismatch: %v", table, err) + } + }) + } +} diff --git a/yb-voyager/src/migassessment/assessmentDB.go b/yb-voyager/src/migassessment/assessmentDB.go index c4700ccb63..687919122a 100644 --- a/yb-voyager/src/migassessment/assessmentDB.go +++ b/yb-voyager/src/migassessment/assessmentDB.go @@ -59,7 +59,7 @@ type TableIndexStats struct { SizeInBytes *int64 `json:"SizeInBytes"` } -func GetSourceMetadataDBFilePath() string { +var GetSourceMetadataDBFilePath = func() string { return filepath.Join(AssessmentDir, "dbs", "assessment.db") } diff --git a/yb-voyager/src/migassessment/assessmentDB_test.go b/yb-voyager/src/migassessment/assessmentDB_test.go new file mode 100644 index 0000000000..843d1cca06 --- /dev/null +++ b/yb-voyager/src/migassessment/assessmentDB_test.go @@ -0,0 +1,131 @@ +package migassessment + +import ( + "database/sql" + "fmt" + "os" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" +) + +func TestInitAssessmentDB(t *testing.T) { + expectedTables := map[string]map[string]testutils.ColumnPropertiesSqlite{ + TABLE_INDEX_IOPS: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "object_name": {Type: "TEXT", PrimaryKey: 2}, + "object_type": {Type: "TEXT"}, + "seq_reads": {Type: "INTEGER"}, + "row_writes": {Type: "INTEGER"}, + "measurement_type": {Type: "TEXT", PrimaryKey: 3}, + }, + TABLE_INDEX_SIZES: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "object_name": {Type: "TEXT", PrimaryKey: 2}, + "object_type": {Type: "TEXT"}, + "size_in_bytes": {Type: "INTEGER"}, + }, + TABLE_ROW_COUNTS: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "table_name": {Type: "TEXT", PrimaryKey: 2}, + "row_count": {Type: "INTEGER"}, + }, + TABLE_COLUMNS_COUNT: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "object_name": {Type: "TEXT", PrimaryKey: 2}, + "object_type": {Type: "TEXT"}, + "column_count": {Type: "INTEGER"}, + }, + INDEX_TO_TABLE_MAPPING: { + "index_schema": {Type: "TEXT", PrimaryKey: 1}, + "index_name": {Type: "TEXT", PrimaryKey: 2}, + "table_schema": {Type: "TEXT"}, + "table_name": {Type: "TEXT"}, + }, + OBJECT_TYPE_MAPPING: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "object_name": {Type: "TEXT", PrimaryKey: 2}, + "object_type": {Type: "TEXT"}, + }, + TABLE_COLUMNS_DATA_TYPES: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "table_name": {Type: "TEXT", PrimaryKey: 2}, + "column_name": {Type: "TEXT", PrimaryKey: 3}, + "data_type": {Type: "TEXT"}, + }, + TABLE_INDEX_STATS: { + "schema_name": {Type: "TEXT", PrimaryKey: 1}, + "object_name": {Type: "TEXT", PrimaryKey: 2}, + "row_count": {Type: "INTEGER"}, + "column_count": {Type: "INTEGER"}, + "reads": {Type: "INTEGER"}, + "writes": {Type: "INTEGER"}, + "reads_per_second": {Type: "INTEGER"}, + "writes_per_second": {Type: "INTEGER"}, + "is_index": {Type: "BOOLEAN"}, + "object_type": {Type: "TEXT"}, + "parent_table_name": {Type: "TEXT"}, + "size_in_bytes": {Type: "INTEGER"}, + }, + DB_QUERIES_SUMMARY: { + "queryid": {Type: "BIGINT"}, + "query": {Type: "TEXT"}, + }, + } + + // Create a temporary SQLite database file for testing + tempFile, err := os.CreateTemp(os.TempDir(), "test_assessment_db_*.db") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + // Ensure the file is removed after the test + defer func() { + err := os.Remove(tempFile.Name()) + if err != nil { + t.Fatalf("Failed to remove temporary file: %v", err) + } + }() + + GetSourceMetadataDBFilePath = func() string { + return tempFile.Name() + } + + err = InitAssessmentDB() + if err != nil { + t.Fatalf("Failed to initialize database: %v", err) + } else { + t.Logf("Database initialized successfully") + } + + // Open the temporary database for verification + db, err := sql.Open("sqlite3", tempFile.Name()) + if err != nil { + t.Fatalf("Failed to open temporary database: %v", err) + } + defer db.Close() + + // Verify the existence of each table and no extra tables + t.Run("Check table existence and no extra tables", func(t *testing.T) { + err := testutils.CheckTableExistenceSqlite(t, db, expectedTables) + if err != nil { + t.Errorf("Table existence mismatch: %v", err) + } + }) + + // Verify the structure of each table + for table, expectedColumns := range expectedTables { + t.Run(fmt.Sprintf("Check structure of %s table", table), func(t *testing.T) { + err := testutils.CheckTableStructureSqlite(db, table, expectedColumns) + if err != nil { + t.Errorf("Table %s structure mismatch: %v", table, err) + } + }) + } + +} + +// Helper function to create a string pointer +func stringPointer(s string) *string { + return &s +} diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index 279ae9a919..0a0951581b 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -3,12 +3,16 @@ package namereg import ( "fmt" "os" + "path/filepath" + "reflect" + "strings" "testing" "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) @@ -377,7 +381,7 @@ func TestDifferentSchemaInSameDBAsSourceReplica2(t *testing.T) { //===================================================== type dummySourceDB struct { - tableNames map[string][]string // schemaName -> tableNames + tableNames map[string][]string // schemaName -> tableNames sequenceNames map[string][]string // schemaName -> sequenceNames } @@ -398,8 +402,8 @@ func (db *dummySourceDB) GetAllSequencesRaw(schemaName string) ([]string, error) } type dummyTargetDB struct { - tableNames map[string][]string // schemaName -> tableNames - sequenceNames map[string][]string // schemaName -> sequenceNames + tableNames map[string][]string // schemaName -> tableNames + sequenceNames map[string][]string // schemaName -> sequenceNames } func (db *dummyTargetDB) GetAllSchemaNamesRaw() ([]string, error) { @@ -414,7 +418,6 @@ func (db *dummyTargetDB) GetAllTableNamesRaw(schemaName string) ([]string, error return tableNames, nil } - func (db *dummyTargetDB) GetAllSequencesRaw(schemaName string) ([]string, error) { sequenceNames, ok := db.sequenceNames[schemaName] if !ok { @@ -448,18 +451,18 @@ func TestNameRegistryWithDummyDBs(t *testing.T) { } sourceNamesMap := make(map[string][]string) - for k,v := range dummySdb.tableNames { + for k, v := range dummySdb.tableNames { sourceNamesMap[k] = append(sourceNamesMap[k], v...) } - for k,v := range dummySdb.sequenceNames { + for k, v := range dummySdb.sequenceNames { sourceNamesMap[k] = append(sourceNamesMap[k], v...) } targetNamesMap := make(map[string][]string) - for k,v := range dummyTdb.tableNames { + for k, v := range dummyTdb.tableNames { targetNamesMap[k] = append(targetNamesMap[k], v...) } - for k,v := range dummyTdb.sequenceNames { + for k, v := range dummyTdb.sequenceNames { targetNamesMap[k] = append(targetNamesMap[k], v...) } @@ -497,7 +500,7 @@ func TestNameRegistryWithDummyDBs(t *testing.T) { seq1 := buildNameTuple(reg, "SAKILA", "SEQ1", "", "") stup, err := reg.LookupTableName("SEQ1") require.Nil(err) - assert.Equal(seq1, stup) + assert.Equal(seq1, stup) // When `export data` restarts, the registry should be reloaded from the file. reg = newNameRegistry("") @@ -549,3 +552,132 @@ func TestNameRegistryWithDummyDBs(t *testing.T) { assert.Equal(table1, ntup) assert.Equal(`SAKILA_FF."TABLE1"`, table1.ForUserQuery()) } + +// Unit tests for breaking changes in NameRegistry. + +func TestNameRegistryStructs(t *testing.T) { + + tests := []struct { + name string + actualType reflect.Type + expectedType interface{} + }{ + { + name: "Validate NameRegistryParams Struct Definition", + actualType: reflect.TypeOf(NameRegistryParams{}), + expectedType: struct { + FilePath string + Role string + SourceDBType string + SourceDBSchema string + SourceDBName string + SDB SourceDBInterface + TargetDBSchema string + YBDB YBDBInterface + }{}, + }, + { + name: "Validate NameRegistry Struct Definition", + actualType: reflect.TypeOf(NameRegistry{}), + expectedType: struct { + SourceDBType string + SourceDBSchemaNames []string + DefaultSourceDBSchemaName string + SourceDBTableNames map[string][]string + YBSchemaNames []string + DefaultYBSchemaName string + YBTableNames map[string][]string + DefaultSourceReplicaDBSchemaName string + params NameRegistryParams + }{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testutils.CompareStructs(t, tt.actualType, reflect.TypeOf(tt.expectedType), tt.name) + }) + } +} + +func TestNameRegistryJson(t *testing.T) { + exportDir := filepath.Join(os.TempDir(), "namereg") + outputFilePath := filepath.Join(exportDir, "test_dummy_name_registry.json") + + // Create a sample NameRegistry instance + reg := &NameRegistry{ + SourceDBType: ORACLE, + params: NameRegistryParams{ + FilePath: outputFilePath, + Role: TARGET_DB_IMPORTER_ROLE, + SourceDBType: ORACLE, + SourceDBSchema: "SAKILA", + SourceDBName: "ORCLPDB1", + TargetDBSchema: "ybsakila", + }, + SourceDBSchemaNames: []string{"SAKILA"}, + DefaultSourceDBSchemaName: "SAKILA", + SourceDBTableNames: map[string][]string{ + "SAKILA": {"TABLE1", "TABLE2", "MixedCaps", "lower_caps"}, + }, + YBSchemaNames: []string{"ybsakila"}, + DefaultYBSchemaName: "ybsakila", + YBTableNames: map[string][]string{ + "ybsakila": {"table1", "table2", "mixedcaps", "lower_caps"}, + }, + DefaultSourceReplicaDBSchemaName: "SAKILA_FF", + } + + // Ensure the export directory exists + if err := os.MkdirAll(exportDir, 0755); err != nil { + t.Fatalf("Failed to create export directory: %v", err) + } + + // Clean up the export directory + defer func() { + if err := os.RemoveAll(exportDir); err != nil { + t.Fatalf("Failed to remove export directory: %v", err) + } + }() + + // Marshal the NameRegistry instance to JSON + err := reg.save() + if err != nil { + t.Fatalf("Failed to save NameRegistry to JSON: %v", err) + } + + // TODO: Use a single string instead of a slice of strings for the expected JSON + expectedJSON := strings.Join([]string{ + "{", + ` "SourceDBType": "oracle",`, + ` "SourceDBSchemaNames": [`, + ` "SAKILA"`, + " ],", + ` "DefaultSourceDBSchemaName": "SAKILA",`, + ` "SourceDBTableNames": {`, + ` "SAKILA": [`, + ` "TABLE1",`, + ` "TABLE2",`, + ` "MixedCaps",`, + ` "lower_caps"`, + " ]", + " },", + ` "YBSchemaNames": [`, + ` "ybsakila"`, + " ],", + ` "DefaultYBSchemaName": "ybsakila",`, + ` "YBTableNames": {`, + ` "ybsakila": [`, + ` "table1",`, + ` "table2",`, + ` "mixedcaps",`, + ` "lower_caps"`, + " ]", + " },", + ` "DefaultSourceReplicaDBSchemaName": "SAKILA_FF"`, + "}", + }, "\n") + + // Read the JSON file and compare it with the expected JSON + testutils.CompareJson(t, outputFilePath, expectedJSON, exportDir) +} diff --git a/yb-voyager/src/testutils/testing_utils.go b/yb-voyager/src/testutils/testing_utils.go new file mode 100644 index 0000000000..2a10007dee --- /dev/null +++ b/yb-voyager/src/testutils/testing_utils.go @@ -0,0 +1,352 @@ +package testutils + +import ( + "database/sql" + "fmt" + "os" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" +) + +type ColumnPropertiesSqlite struct { + Type string // Data type (e.g., INTEGER, TEXT) + PrimaryKey int // Whether it's a primary key values can be 0,1,2,3. If 1 then it is primary key. 2,3 etc. are used for composite primary key + NotNull bool // Whether the column has a NOT NULL constraint + Default sql.NullString // Default value, if any (nil means no default) +} + +// Column represents a column's expected metadata +type ColumnPropertiesPG struct { + Type string + IsPrimary bool + IsNullable string + Default sql.NullString +} + +// CompareStructs compares two struct types and reports any mismatches. +func CompareStructs(t *testing.T, actual, expected reflect.Type, structName string) { + assert.Equal(t, reflect.Struct, actual.Kind(), "%s: Actual type must be a struct. There is some breaking change!", structName) + assert.Equal(t, reflect.Struct, expected.Kind(), "%s: Expected type must be a struct. There is some breaking change!", structName) + + for i := 0; i < max(actual.NumField(), expected.NumField()); i++ { + var actualField, expectedField reflect.StructField + var actualExists, expectedExists bool + + if i < actual.NumField() { + actualField = actual.Field(i) + actualExists = true + } + if i < expected.NumField() { + expectedField = expected.Field(i) + expectedExists = true + } + + // Assert field names match + if actualExists && expectedExists { + assert.Equal(t, expectedField.Name, actualField.Name, "%s: Field name mismatch at position %d. There is some breaking change!", structName, i) + assert.Equal(t, expectedField.Type.String(), actualField.Type.String(), "%s: Field type mismatch for field %s. There is some breaking change!", structName, expectedField.Name) + assert.Equal(t, expectedField.Tag, actualField.Tag, "%s: Field tag mismatch for field %s. There is some breaking change!", structName, expectedField.Name) + } + + // Report missing or extra fields + if !actualExists && expectedExists { + t.Errorf("%s: Missing field %s of type %s. There is some breaking change!", structName, expectedField.Name, expectedField.Type) + } + if actualExists && !expectedExists { + t.Errorf("%s: Unexpected field %s of type %s. There is some breaking change!", structName, actualField.Name, actualField.Type) + } + } +} + +// CompareJson compares two structs by marshalling them into JSON and reports any differences. +func CompareJson(t *testing.T, outputFilePath string, expectedJSON string, exportDir string) { + // Read the output JSON file + outputBytes, err := os.ReadFile(outputFilePath) + if err != nil { + t.Fatalf("Failed to read output JSON file: %v", err) + } + + // Compare the output JSON with the expected JSON + if diff := cmp.Diff(expectedJSON, string(outputBytes)); diff != "" { + t.Errorf("JSON file mismatch (-expected +actual):\n%s", diff) + } + + // Can be used if we don't want to compare pretty printed JSON + // assert.JSONEqf(t, expectedJSON, string(outputBytes), "JSON file mismatch. There is some breaking change!") +} + +// Helper function to check table structure +func CheckTableStructureSqlite(db *sql.DB, tableName string, expectedColumns map[string]ColumnPropertiesSqlite) error { + // Query to get table info + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s);", tableName)) + if err != nil { + return fmt.Errorf("failed to get table info for %s: %w", tableName, err) + } + defer rows.Close() + + // Check if columns match expected ones + actualColumns := make(map[string]ColumnPropertiesSqlite) + for rows.Next() { + var cid int + var name string + var cp ColumnPropertiesSqlite + if err := rows.Scan(&cid, &name, &cp.Type, &cp.NotNull, &cp.Default, &cp.PrimaryKey); err != nil { + return err + } + actualColumns[name] = ColumnPropertiesSqlite{ + Type: cp.Type, + PrimaryKey: cp.PrimaryKey, + NotNull: cp.NotNull, + Default: cp.Default, + } + } + + // Compare actual columns with expected columns + for colName, expectedProps := range expectedColumns { + actualProps, exists := actualColumns[colName] + if !exists { + return fmt.Errorf("table %s missing expected column: %s. There is some breaking change!", tableName, colName) + } + + // Check type + if actualProps.Type != expectedProps.Type { + return fmt.Errorf("table %s column %s: expected type %s, got %s. There is some breaking change!", tableName, colName, expectedProps.Type, actualProps.Type) + } + + // Check if it's part of the primary key + if actualProps.PrimaryKey != expectedProps.PrimaryKey { + return fmt.Errorf("table %s column %s: expected primary key to be %v, got %v. There is some breaking change!", tableName, colName, expectedProps.PrimaryKey, actualProps.PrimaryKey) + } + + // Check NOT NULL constraint + if actualProps.NotNull != expectedProps.NotNull { + return fmt.Errorf("table %s column %s: expected NOT NULL to be %v, got %v. There is some breaking change!", tableName, colName, expectedProps.NotNull, actualProps.NotNull) + } + + // Check default value + if (expectedProps.Default.Valid && !actualProps.Default.Valid) || (!expectedProps.Default.Valid && actualProps.Default.Valid) || (expectedProps.Default.Valid && actualProps.Default.Valid && expectedProps.Default.String != actualProps.Default.String) { + return fmt.Errorf("table %s column %s: expected default value %v, got %v. There is some breaking change!", tableName, colName, expectedProps.Default, actualProps.Default) + } + } + + // Check for any additional unexpected columns + for colName := range actualColumns { + if _, exists := expectedColumns[colName]; !exists { + return fmt.Errorf("table %s has unexpected additional column: %s. There is some breaking change!", tableName, colName) + } + } + + return nil +} + +// Helper function to check table structure +func CheckTableStructurePG(t *testing.T, db *sql.DB, schema, table string, expectedColumns map[string]ColumnPropertiesPG) { + queryColumns := ` + SELECT column_name, data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_name = $1;` + + rows, err := db.Query(queryColumns, table) + if err != nil { + t.Fatalf("Failed to query columns for table %s.%s: %v", schema, table, err) + } + defer rows.Close() + + actualColumns := make(map[string]ColumnPropertiesPG) + for rows.Next() { + var colName string + var col ColumnPropertiesPG + err := rows.Scan(&colName, &col.Type, &col.IsNullable, &col.Default) + if err != nil { + t.Fatalf("Failed to scan column metadata: %v", err) + } + actualColumns[colName] = col + } + + // Compare columns + for colName, expectedProps := range expectedColumns { + actual, found := actualColumns[colName] + if !found { + t.Errorf("Missing expected column in table %s.%s: %s.\nThere is some breaking change!", schema, table, colName) + continue + } + if actual.Type != expectedProps.Type || actual.IsNullable != expectedProps.IsNullable || actual.Default != expectedProps.Default { + t.Errorf("Column mismatch in table %s.%s: \nexpected %+v, \ngot %+v.\nThere is some breaking change!", schema, table, expectedProps, actual) + } + } + + // Check for extra columns + for actualName := range actualColumns { + found := false + for expectedName, _ := range expectedColumns { + if actualName == expectedName { + found = true + break + } + } + if !found { + t.Errorf("Unexpected column in table %s.%s: %s.\nThere is some breaking change!", schema, table, actualName) + } + } + + // Check primary keys + checkPrimaryKeyOfTablePG(t, db, schema, table, expectedColumns) +} + +func checkPrimaryKeyOfTablePG(t *testing.T, db *sql.DB, schema, table string, expectedColumns map[string]ColumnPropertiesPG) { + // Validate primary keys + queryPrimaryKeys := ` + SELECT conrelid::regclass AS table_name, + conname AS primary_key, + pg_get_constraintdef(oid) + FROM pg_constraint + WHERE contype = 'p' -- 'p' indicates primary key + AND conrelid::regclass::text = $1 + ORDER BY conrelid::regclass::text, contype DESC;` + + rows, err := db.Query(queryPrimaryKeys, fmt.Sprintf("%s.%s", schema, table)) + if err != nil { + t.Fatalf("Failed to query primary keys for table %s.%s: %v", schema, table, err) + } + defer rows.Close() + + // Map to store primary key columns (not just the constraint name) + // Output is like: + // table_name | primary_key | primary_key_definition + // ybvoyager_metadata.ybvoyager_import_data_batches_metainfo_v3 | ybvoyager_import_data_batches_metainfo_v3_pkey | PRIMARY KEY (migration_uuid, data_file_name, batch_number, schema_name, table_name) + primaryKeyColumns := map[string]bool{} + for rows.Next() { + var tableName, pk, constraintDef string + if err := rows.Scan(&tableName, &pk, &constraintDef); err != nil { + t.Fatalf("Failed to scan primary key: %v", err) + } + + // Parse the columns from the constraint definition (e.g., "PRIMARY KEY (col1, col2, ...)") + columns := parsePrimaryKeyColumnsPG(constraintDef) + for _, col := range columns { + primaryKeyColumns[col] = true + } + } + + // Check if the primary key columns match the expected primary key columns + for expectedName, expectedParams := range expectedColumns { + if expectedParams.IsPrimary { + if _, found := primaryKeyColumns[expectedName]; !found { + t.Errorf("Missing expected primary key column in table %s.%s: %s.\nThere is some breaking change!", schema, table, expectedName) + } + } + } + + // Check if there are any extra primary key columns + for col := range primaryKeyColumns { + found := false + for expectedName, expectedParams := range expectedColumns { + if expectedName == col && expectedParams.IsPrimary { + found = true + break + } + } + if !found { + t.Errorf("Unexpected primary key column in table %s.%s: %s.\nThere is some breaking change!", schema, table, col) + } + } +} + +// Helper function to parse primary key columns from the constraint definition +func parsePrimaryKeyColumnsPG(constraintDef string) []string { + // Define the regex pattern + re := regexp.MustCompile(`PRIMARY KEY\s*\((.*?)\)`) + + // Extract the column list inside "PRIMARY KEY(...)" + matches := re.FindStringSubmatch(constraintDef) + if len(matches) < 2 { + return nil // Return nil if no match is found + } + + // Split by commas to get individual column names + columns := strings.Split(matches[1], ",") + for i := range columns { + columns[i] = strings.TrimSpace(columns[i]) // Remove extra spaces around column names + } + + return columns +} + +// Helper function to check table existence and no extra tables +func CheckTableExistenceSqlite(t *testing.T, db *sql.DB, expectedTables map[string]map[string]ColumnPropertiesSqlite) error { + // Query to get table names + rows, err := db.Query("SELECT name FROM sqlite_master WHERE type='table';") + if err != nil { + return fmt.Errorf("failed to get table names: %w", err) + } + defer rows.Close() + + // Check if tables match expected ones + actualTables := make(map[string]struct{}) + for rows.Next() { + var tableName string + if err := rows.Scan(&tableName); err != nil { + return err + } + actualTables[tableName] = struct{}{} + } + + // Compare actual tables with expected tables + for tableName := range expectedTables { + if _, exists := actualTables[tableName]; !exists { + return fmt.Errorf("expected table %s not found. There is some breaking change!", tableName) + } else { + t.Logf("Found table: %s", tableName) + } + } + + // Check for any additional unexpected tables + for tableName := range actualTables { + if _, exists := expectedTables[tableName]; !exists { + return fmt.Errorf("unexpected additional table: %s. There is some breaking change!", tableName) + } + } + + return nil +} + +// validateSchema validates the schema, tables, and columns +func CheckTableExistencePG(t *testing.T, db *sql.DB, schema string, expectedTables map[string]map[string]ColumnPropertiesPG) { + // Check all tables in the schema + queryTables := `SELECT table_schema || '.' || table_name AS qualified_table_name + FROM information_schema.tables + WHERE table_schema = $1;` + rows, err := db.Query(queryTables, schema) + if err != nil { + t.Fatalf("Failed to query tables in schema %s: %v", schema, err) + } + defer rows.Close() + + actualTables := make(map[string]bool) + for rows.Next() { + var tableName string + if err := rows.Scan(&tableName); err != nil { + t.Fatalf("Failed to scan table name: %v", err) + } + actualTables[tableName] = true + } + + // Compare tables + for expectedTable := range expectedTables { + if !actualTables[expectedTable] { + t.Errorf("Missing expected table: %s", expectedTable) + } + } + + // Check for extra tables + for actualTable := range actualTables { + if _, found := expectedTables[actualTable]; !found { + t.Errorf("Unexpected table found: %s", actualTable) + } + } +} diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go new file mode 100644 index 0000000000..251ced5a63 --- /dev/null +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -0,0 +1,82 @@ +package tgtdb + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" +) + +func TestCreateVoyagerSchemaPG(t *testing.T) { + ctx := context.Background() + + // Start a PostgreSQL container + pgContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.POSTGRESQL) + assert.NoError(t, err, "Failed to start PostgreSQL container") + defer pgContainer.Terminate(ctx) + + // Connect to the database + dsn := fmt.Sprintf("host=%s port=%s user=testuser password=testpassword dbname=testdb sslmode=disable", host, port.Port()) + db, err := sql.Open("postgres", dsn) + assert.NoError(t, err) + defer db.Close() + + // Wait for the database to be ready + err = testcontainers.WaitForDBToBeReady(db) + assert.NoError(t, err) + + // Initialize the TargetYugabyteDB instance + pg := &TargetPostgreSQL{ + db: db, + } + + // Call CreateVoyagerSchema + err = pg.CreateVoyagerSchema() + assert.NoError(t, err, "CreateVoyagerSchema failed") + + expectedTables := map[string]map[string]testutils.ColumnPropertiesPG{ + BATCH_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "data_file_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "batch_number": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "schema_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "rows_imported": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + EVENT_CHANNELS_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "channel_no": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "last_applied_vsn": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_inserts": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_deletes": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_updates": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + EVENTS_PER_TABLE_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "channel_no": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "total_events": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_inserts": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_deletes": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_updates": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + } + + // Validate the schema and tables + t.Run("Check all the expected tables and no extra tables", func(t *testing.T) { + testutils.CheckTableExistencePG(t, db, BATCH_METADATA_TABLE_SCHEMA, expectedTables) + }) + + // Validate columns for each table + for tableName, expectedColumns := range expectedTables { + t.Run(fmt.Sprintf("Check columns for %s table", tableName), func(t *testing.T) { + table := strings.Split(tableName, ".")[1] + testutils.CheckTableStructurePG(t, db, BATCH_METADATA_TABLE_SCHEMA, table, expectedColumns) + }) + } +} diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go new file mode 100644 index 0000000000..af22664460 --- /dev/null +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -0,0 +1,82 @@ +package tgtdb + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" +) + +func TestCreateVoyagerSchemaYB(t *testing.T) { + ctx := context.Background() + + // Start a YugabyteDB container + ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) + assert.NoError(t, err, "Failed to start YugabyteDB container") + defer ybContainer.Terminate(ctx) + + // Connect to the database + dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) + db, err := sql.Open("postgres", dsn) + assert.NoError(t, err) + defer db.Close() + + // Wait for the database to be ready + err = testcontainers.WaitForDBToBeReady(db) + assert.NoError(t, err) + + // Initialize the TargetYugabyteDB instance + yb := &TargetYugabyteDB{ + db: db, + } + + // Call CreateVoyagerSchema + err = yb.CreateVoyagerSchema() + assert.NoError(t, err, "CreateVoyagerSchema failed") + + expectedTables := map[string]map[string]testutils.ColumnPropertiesPG{ + BATCH_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "data_file_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "batch_number": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "schema_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "rows_imported": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + EVENT_CHANNELS_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "channel_no": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "last_applied_vsn": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_inserts": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_deletes": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_updates": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + EVENTS_PER_TABLE_METADATA_TABLE_NAME: { + "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "channel_no": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "total_events": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_inserts": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_deletes": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + "num_updates": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, + }, + } + + // Validate the schema and tables + t.Run("Check all the expected tables and no extra tables", func(t *testing.T) { + testutils.CheckTableExistencePG(t, db, BATCH_METADATA_TABLE_SCHEMA, expectedTables) + }) + + // Validate columns for each table + for tableName, expectedColumns := range expectedTables { + t.Run(fmt.Sprintf("Check columns for %s table", tableName), func(t *testing.T) { + table := strings.Split(tableName, ".")[1] + testutils.CheckTableStructurePG(t, db, BATCH_METADATA_TABLE_SCHEMA, table, expectedColumns) + }) + } +} diff --git a/yb-voyager/testcontainers/testcontainers.go b/yb-voyager/testcontainers/testcontainers.go new file mode 100644 index 0000000000..336d7b2e30 --- /dev/null +++ b/yb-voyager/testcontainers/testcontainers.go @@ -0,0 +1,108 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/docker/go-connections/nat" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + POSTGRESQL = "postgresql" + YUGABYTEDB = "yugabytedb" +) + +func StartDBContainer(ctx context.Context, dbType string) (testcontainers.Container, string, nat.Port, error) { + switch dbType { + case POSTGRESQL: + return startPostgresContainer(ctx) + case YUGABYTEDB: + return startYugabyteDBContainer(ctx) + default: + return nil, "", "", fmt.Errorf("unsupported database type: %s", dbType) + } +} + +func startPostgresContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { + // Create a PostgreSQL TestContainer + req := testcontainers.ContainerRequest{ + Image: "postgres:latest", // Use the latest PostgreSQL image + ExposedPorts: []string{"5432/tcp"}, + Env: map[string]string{ + "POSTGRES_USER": "testuser", // Set PostgreSQL username + "POSTGRES_PASSWORD": "testpassword", // Set PostgreSQL password + "POSTGRES_DB": "testdb", // Set PostgreSQL database name + }, + WaitingFor: wait.ForListeningPort("5432/tcp").WithStartupTimeout(30 * 1e9), // Wait for PostgreSQL to be ready + } + + // Start the container + pgContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + + if err != nil { + return nil, "", "", err + } + + // Get the container's host and port + host, err = pgContainer.Host(ctx) + if err != nil { + return nil, "", "", err + } + + port, err = pgContainer.MappedPort(ctx, "5432") + if err != nil { + return nil, "", "", err + } + + return pgContainer, host, port, nil +} + +func startYugabyteDBContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { + // Create a YugabyteDB TestContainer + req := testcontainers.ContainerRequest{ + Image: "yugabytedb/yugabyte:latest", + ExposedPorts: []string{"5433/tcp"}, + WaitingFor: wait.ForListeningPort("5433/tcp"), + Cmd: []string{"bin/yugabyted", "start", "--daemon=false"}, + } + + // Start the container + ybContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, "", "", err + } + + // Get the container's host and port + host, err = ybContainer.Host(ctx) + if err != nil { + return nil, "", "", err + } + + port, err = ybContainer.MappedPort(ctx, "5433") + if err != nil { + return nil, "", "", err + } + + return ybContainer, host, port, nil +} + +// waitForDBConnection waits until the database is ready for connections. +func WaitForDBToBeReady(db *sql.DB) error { + for i := 0; i < 12; i++ { + if err := db.Ping(); err == nil { + return nil + } + time.Sleep(5 * time.Second) + } + return fmt.Errorf("database did not become ready in time") +} From b8f6c849d3c7d81400382338c5834ffc6a97a15a Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Fri, 6 Dec 2024 14:11:00 +0530 Subject: [PATCH 040/105] Changed the expected values wrt Ora2pg export in oracle/sequences test (#2040) --- migtests/tests/oracle/sequences/validate | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/migtests/tests/oracle/sequences/validate b/migtests/tests/oracle/sequences/validate index 3abd78a3b4..8e07b1e988 100755 --- a/migtests/tests/oracle/sequences/validate +++ b/migtests/tests/oracle/sequences/validate @@ -39,14 +39,14 @@ EXPECTED_TABLE_SUM = { } EXPECTED_TABLE_SUM_AFTER_INSERT = { - 'identity_demo_generated_by_def_inc_by': 2227, - 'identity_demo_generated_always': 25, - 'identity_demo_generated_by_def': 30, - 'identity_demo_generated_always_start_with': 223, - 'identity_demo_generated_by_def_st_with_inc_by': 2241, - 'identity_demo_generated_by_def_start_with': 327, - 'identity_demo_with_null': 25, - 'case_sensitive_always': 25 + 'identity_demo_generated_by_def_inc_by': 409, + 'identity_demo_generated_always': 7, + 'identity_demo_generated_by_def': 11, + 'identity_demo_generated_always_start_with': 204, + 'identity_demo_generated_by_def_st_with_inc_by': 423, + 'identity_demo_generated_by_def_start_with': 309, + 'identity_demo_with_null': 7, + 'case_sensitive_always': 7 } if os.environ.get('BETA_FAST_DATA_EXPORT') == '1': From f3b63ce3ace3ce521dfc02b371baff033f89e15c Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Fri, 6 Dec 2024 14:27:24 +0530 Subject: [PATCH 041/105] Sample Issue Tests against YB Versions (#2033) Workflow that will help us detect when issues are fixed in YB: - Whenever we introduce a new issue, write a unit test asserting that it fails. - The unit test will be run against all the latest YB versions in every series that we support (2.18,2.20,2.23,2024.1,2024.2 for example) - YB fixes an issue in 2025.1.0.0. - As part of qualifying voyager for 2025.1 release, all unit tests are also run against 2025.1. Since the issue was fixed, "assert that it fails" will fail. - This prompts us to a. Add MinimumVersionsFixedIn[2025.1] = 2025.1.0.0 to the issue. b. Write a unit test that asserts that parserDetector.GetAllIssues(..., 2025.1) does not return an issue of that type. - The failing unit test will now start passing because it takes into account MinimumVersionsFixedIn[2025.1]. (this part is not implemented yet) --- .github/workflows/go.yml | 2 +- .github/workflows/issue-tests.yml | 50 ++++++++ yb-voyager/go.mod | 5 +- yb-voyager/go.sum | 8 ++ yb-voyager/src/queryissue/ddl_issues_test.go | 121 +++++++++++++++++++ 5 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/issue-tests.yml create mode 100644 yb-voyager/src/queryissue/ddl_issues_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 09d8cddc52..505cc265c8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v ./... + go test -v -skip 'TestDDLIssuesInYBVersion' ./... - name: Vet run: | diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml new file mode 100644 index 0000000000..94e95b16fc --- /dev/null +++ b/.github/workflows/issue-tests.yml @@ -0,0 +1,50 @@ +name: Go + +on: + push: + branches: ['main', '*.*-dev', '*.*.*-dev'] + pull_request: + branches: [main] + +jobs: + + test-issues-against-all-yb-versions: + strategy: + matrix: + version: [2.23.1.0-b220, 2024.1.3.1-b8, 2.20.8.0-b53, 2.18.9.0-b17] + env: + YB_VERSION: ${{ matrix.version }} + YB_CONN_STR: "postgres://yugabyte:yugabyte@127.0.0.1:5433/yugabyte" + + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.23.1" + + - name: Setup YugabyteDB + run: | + # using s3 release instead of docker image to allow testing against un-released versions + wget https://s3.us-west-2.amazonaws.com/releases.yugabyte.com/${YB_VERSION}/yugabyte-${YB_VERSION}-centos-x86_64.tar.gz + mkdir -p yugabyte-${YB_VERSION} + tar -xvzf yugabyte-${YB_VERSION}-centos-x86_64.tar.gz -C yugabyte-${YB_VERSION} --strip-components=1 + yugabyte-${YB_VERSION}/bin/yugabyted start --advertise_address 127.0.0.1 + sleep 20 + + - name: Test YugabyteDB connection + run: | + psql "${YB_CONN_STR}" -c "SELECT version();" + + - name: Build + run: | + cd yb-voyager + go build -v ./... + + - name: Test Issues Against YB Version + run: | + cd yb-voyager + go test -v -run '^TestDDLIssuesInYBVersion$' ./... + diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index b761c2d4a7..93e64b1e7f 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -11,6 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 github.com/davecgh/go-spew v1.1.1 + github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 github.com/fergusstrange/embedded-postgres v1.29.0 @@ -23,6 +24,7 @@ require ( github.com/jackc/pgconn v1.13.0 github.com/jackc/pgx/v4 v4.17.2 github.com/jackc/pgx/v5 v5.0.3 + github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 github.com/mitchellh/go-ps v1.0.0 @@ -35,6 +37,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tebeka/atexit v0.3.0 github.com/testcontainers/testcontainers-go v0.34.0 + github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0 github.com/vbauerster/mpb/v8 v8.4.0 gocloud.dev v0.29.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa @@ -54,7 +57,6 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v27.1.1+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -64,7 +66,6 @@ require ( github.com/jackc/puddle v1.3.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.17.4 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index a0de897915..1369ff42c2 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1127,6 +1127,7 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -1259,6 +1260,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgv github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hanwen/go-fuse/v2 v2.2.0/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -1921,6 +1924,8 @@ github.com/tebeka/atexit v0.3.0/go.mod h1:WJmSUSmMT7WoR7etUOaGBVXk+f5/ZJ+67qwued github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= +github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0 h1:9wIqSZJwBr4s8Q7R3S+rhe1J2zqHHxH0S1bN17ld+CI= +github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0/go.mod h1:bgHrbdYjpNPSstf8HfxChUxc6XztBCSoqDR0syb1Oeg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= @@ -1965,6 +1970,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yugabyte/gocql v1.6.0-yb-1 h1:3anNiHsJwKQ8Dn7RdmkTEuIzV1l7e9QJZ8wkOZ87ELg= +github.com/yugabyte/gocql v1.6.0-yb-1/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2942,6 +2949,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/yb-voyager/src/queryissue/ddl_issues_test.go b/yb-voyager/src/queryissue/ddl_issues_test.go new file mode 100644 index 0000000000..105f565875 --- /dev/null +++ b/yb-voyager/src/queryissue/ddl_issues_test.go @@ -0,0 +1,121 @@ +/* +Copyright (c) YugabyteDB, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryissue + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/testcontainers/testcontainers-go/modules/yugabytedb" +) + +var ( + yugabytedbContainer *yugabytedb.Container + yugabytedbConnStr string + versions = []string{} +) + +func getConn() (*pgx.Conn, error) { + ctx := context.Background() + var connStr string + var err error + if yugabytedbConnStr != "" { + connStr = yugabytedbConnStr + } else { + connStr, err = yugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") + if err != nil { + return nil, err + } + } + + conn, err := pgx.Connect(ctx, connStr) + if err != nil { + return nil, err + } + + return conn, nil +} + +func testXMLFunctionIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, "SELECT xmlconcat('', 'foo')") + assert.ErrorContains(t, err, "unsupported XML feature") +} + +func testStoredGeneratedFunctionsIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE rectangles ( + id SERIAL PRIMARY KEY, + length NUMERIC NOT NULL, + width NUMERIC NOT NULL, + area NUMERIC GENERATED ALWAYS AS (length * width) STORED + )`) + assert.ErrorContains(t, err, "syntax error") +} + +func testUnloggedTableIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, "CREATE UNLOGGED TABLE unlogged_table (a int)") + assert.ErrorContains(t, err, "UNLOGGED database object not supported yet") +} + +func TestDDLIssuesInYBVersion(t *testing.T) { + ybVersion := os.Getenv("YB_VERSION") + if ybVersion == "" { + panic("YB_VERSION env variable is not set. Set YB_VERSIONS=2024.1.3.0-b105 for example") + } + + yugabytedbConnStr = os.Getenv("YB_CONN_STR") + if yugabytedbConnStr == "" { + // spawn yugabytedb container + var err error + ctx := context.Background() + yugabytedbContainer, err = yugabytedb.Run( + ctx, + "yugabytedb/yugabyte:"+ybVersion, + ) + assert.NoError(t, err) + defer yugabytedbContainer.Terminate(context.Background()) + } + + // run tests + var success bool + success = t.Run(fmt.Sprintf("%s-%s", "xml functions", ybVersion), testXMLFunctionIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "stored generated functions", ybVersion), testStoredGeneratedFunctionsIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "unlogged table", ybVersion), testUnloggedTableIssue) + assert.True(t, success) + +} From 9ca3f36e02280cc561128ca8bc8058f61984a388 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 6 Dec 2024 17:51:04 +0530 Subject: [PATCH 042/105] Fix expected file for mgi pg schema for MinimumVersionsFixedIn field in Unsupported PL/pgSQL objects (#2039) --- .../tests/pg/mgi/expected_files/expectedAssessmentReport.json | 1 + 1 file changed, 1 insertion(+) diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index ee6799397f..8e10a3d80d 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -13762,6 +13762,7 @@ "SqlStatement": "acc_accession.accID%TYPE" } ], + "MinimumVersionsFixedIn": null, "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported" } ] From c69f44376647ea100788545ffa07888d1c47240b Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Tue, 10 Dec 2024 14:50:25 +0530 Subject: [PATCH 043/105] Refactor issue, queryissue pkgs (#2044) - Move issue definitions from issue pkg into queryissue pkg - Move IssueInstance from issue pkg into queryIssue pkg. - Rename IssueInstance to queryIssue - Put queryissue and queryparser under a common folder query --- yb-voyager/cmd/analyzeSchema.go | 7 +- yb-voyager/cmd/assessMigrationCommand.go | 4 +- yb-voyager/cmd/importSchemaYugabyteDB.go | 2 +- yb-voyager/src/issue/issue.go | 18 -- yb-voyager/src/issue/issue_test.go | 8 +- .../{issue => query/queryissue}/constants.go | 2 +- .../queryissue/detectors.go} | 18 +- .../queryissue/detectors_ddl.go} | 179 ++++++------ .../queryissue/detectors_test.go} | 14 +- .../src/{ => query}/queryissue/helpers.go | 0 .../ddl.go => query/queryissue/issues_ddl.go} | 259 +++++++++--------- .../queryissue/issues_ddl_test.go} | 0 .../dml.go => query/queryissue/issues_dml.go} | 22 +- .../queryissue/parser_issue_detector.go} | 51 ++-- .../src/query/queryissue/query_issue.go | 42 +++ .../{ => query}/queryparser/ddl_processor.go | 0 .../queryparser/helpers_protomsg.go | 5 + .../{ => query}/queryparser/helpers_struct.go | 8 +- .../queryparser/object_collector.go | 0 .../{ => query}/queryparser/query_parser.go | 11 + .../queryparser/traversal_plpgsql.go | 0 .../queryparser/traversal_proto.go} | 0 .../{ => query}/queryparser/traversal_test.go | 0 23 files changed, 339 insertions(+), 311 deletions(-) rename yb-voyager/src/{issue => query/queryissue}/constants.go (99%) rename yb-voyager/src/{queryissue/unsupported_dml_constructs.go => query/queryissue/detectors.go} (91%) rename yb-voyager/src/{queryissue/unsupported_ddls.go => query/queryissue/detectors_ddl.go} (75%) rename yb-voyager/src/{queryissue/unsupported_dml_constructs_test.go => query/queryissue/detectors_test.go} (96%) rename yb-voyager/src/{ => query}/queryissue/helpers.go (100%) rename yb-voyager/src/{issue/ddl.go => query/queryissue/issues_ddl.go} (56%) rename yb-voyager/src/{queryissue/ddl_issues_test.go => query/queryissue/issues_ddl_test.go} (100%) rename yb-voyager/src/{issue/dml.go => query/queryissue/issues_dml.go} (71%) rename yb-voyager/src/{queryissue/queryissue.go => query/queryissue/parser_issue_detector.go} (87%) create mode 100644 yb-voyager/src/query/queryissue/query_issue.go rename yb-voyager/src/{ => query}/queryparser/ddl_processor.go (100%) rename yb-voyager/src/{ => query}/queryparser/helpers_protomsg.go (98%) rename yb-voyager/src/{ => query}/queryparser/helpers_struct.go (98%) rename yb-voyager/src/{ => query}/queryparser/object_collector.go (100%) rename yb-voyager/src/{ => query}/queryparser/query_parser.go (69%) rename yb-voyager/src/{ => query}/queryparser/traversal_plpgsql.go (100%) rename yb-voyager/src/{queryparser/traversal.go => query/queryparser/traversal_proto.go} (100%) rename yb-voyager/src/{ => query}/queryparser/traversal_test.go (100%) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index a60f0e8ae9..33c45caac4 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -35,10 +35,9 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" - "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryissue" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryissue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" @@ -605,7 +604,7 @@ var MigrationCaveatsIssues = []string{ UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB, } -func convertIssueInstanceToAnalyzeIssue(issueInstance issue.IssueInstance, fileName string, isPlPgSQLIssue bool) utils.Issue { +func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.Issue { issueType := UNSUPPORTED_FEATURES switch true { case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f15c43a0f7..b158ef8f15 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -41,8 +41,8 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryissue" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryissue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" diff --git a/yb-voyager/cmd/importSchemaYugabyteDB.go b/yb-voyager/cmd/importSchemaYugabyteDB.go index e651d0a5f5..ecdf28446b 100644 --- a/yb-voyager/cmd/importSchemaYugabyteDB.go +++ b/yb-voyager/cmd/importSchemaYugabyteDB.go @@ -27,7 +27,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index 5f5dae6823..c30d4f8372 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -40,21 +40,3 @@ func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { } return v.GreaterThanOrEqual(minVersionFixedInSeries), nil } - -type IssueInstance struct { - Issue - ObjectType string // TABLE, FUNCTION, DML_QUERY? - ObjectName string // table name/function name/etc - SqlStatement string - Details map[string]interface{} // additional details about the issue -} - -func newIssueInstance(issue Issue, objectType string, objectName string, sqlStatement string, details map[string]interface{}) IssueInstance { - return IssueInstance{ - Issue: issue, - ObjectType: objectType, - ObjectName: objectName, - SqlStatement: sqlStatement, - Details: details, - } -} diff --git a/yb-voyager/src/issue/issue_test.go b/yb-voyager/src/issue/issue_test.go index 97042daf2b..19ae50de6d 100644 --- a/yb-voyager/src/issue/issue_test.go +++ b/yb-voyager/src/issue/issue_test.go @@ -27,7 +27,7 @@ func TestIssueFixedInStable(t *testing.T) { fixedVersion, err := ybversion.NewYBVersion("2024.1.1.0") assert.NoError(t, err) issue := Issue{ - Type: ADVISORY_LOCKS, + Type: "ADVISORY_LOCKS", MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ ybversion.SERIES_2024_1: fixedVersion, }, @@ -52,7 +52,7 @@ func TestIssueFixedInPreview(t *testing.T) { fixedVersion, err := ybversion.NewYBVersion("2.21.4.5") assert.NoError(t, err) issue := Issue{ - Type: ADVISORY_LOCKS, + Type: "ADVISORY_LOCKS", MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ ybversion.SERIES_2_21: fixedVersion, }, @@ -80,7 +80,7 @@ func TestIssueFixedInStableOld(t *testing.T) { assert.NoError(t, err) issue := Issue{ - Type: ADVISORY_LOCKS, + Type: "ADVISORY_LOCKS", MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ ybversion.SERIES_2024_1: fixedVersionStable, ybversion.SERIES_2_20: fixedVersionStableOld, @@ -107,7 +107,7 @@ func TestIssueFixedInStableOld(t *testing.T) { func TestIssueFixedFalseWhenMinimumNotSpecified(t *testing.T) { issue := Issue{ - Type: ADVISORY_LOCKS, + Type: "ADVISORY_LOCKS", } versionsToCheck := []string{"2024.1.0.0", "2.20.7.4", "2.21.1.1"} diff --git a/yb-voyager/src/issue/constants.go b/yb-voyager/src/query/queryissue/constants.go similarity index 99% rename from yb-voyager/src/issue/constants.go rename to yb-voyager/src/query/queryissue/constants.go index 6599de0fae..49fdf5423d 100644 --- a/yb-voyager/src/issue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package issue +package queryissue // Types const ( diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs.go b/yb-voyager/src/query/queryissue/detectors.go similarity index 91% rename from yb-voyager/src/queryissue/unsupported_dml_constructs.go rename to yb-voyager/src/query/queryissue/detectors.go index fb507e7c89..a89da8c544 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -19,13 +19,13 @@ import ( log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) const ( - ADVISORY_LOCKS = "Advisory Locks" - SYSTEM_COLUMNS = "System Columns" - XML_FUNCTIONS = "XML Functions" + ADVISORY_LOCKS_NAME = "Advisory Locks" + SYSTEM_COLUMNS_NAME = "System Columns" + XML_FUNCTIONS_NAME = "XML Functions" ) // To Add a new unsupported query construct implement this interface for all possible nodes for that construct @@ -42,10 +42,10 @@ type FuncCallDetector struct { func NewFuncCallDetector() *FuncCallDetector { unsupportedFuncs := make(map[string]string) for _, fname := range unsupportedAdvLockFuncs { - unsupportedFuncs[fname] = ADVISORY_LOCKS + unsupportedFuncs[fname] = ADVISORY_LOCKS_NAME } for _, fname := range unsupportedXmlFunctions { - unsupportedFuncs[fname] = XML_FUNCTIONS + unsupportedFuncs[fname] = XML_FUNCTIONS_NAME } return &FuncCallDetector{ @@ -75,7 +75,7 @@ type ColumnRefDetector struct { func NewColumnRefDetector() *ColumnRefDetector { unsupportedColumns := make(map[string]string) for _, colName := range unsupportedSysCols { - unsupportedColumns[colName] = SYSTEM_COLUMNS + unsupportedColumns[colName] = SYSTEM_COLUMNS_NAME } return &ColumnRefDetector{ @@ -108,7 +108,7 @@ func NewXmlExprDetector() *XmlExprDetector { func (d *XmlExprDetector) Detect(msg protoreflect.Message) ([]string, error) { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_XMLEXPR_NODE { log.Debug("detected xml expression") - return []string{XML_FUNCTIONS}, nil + return []string{XML_FUNCTIONS_NAME}, nil } return nil, nil } @@ -136,7 +136,7 @@ func NewRangeTableFuncDetector() *RangeTableFuncDetector { func (d *RangeTableFuncDetector) Detect(msg protoreflect.Message) ([]string, error) { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_RANGETABLEFUNC_NODE { if queryparser.IsXMLTable(msg) { - return []string{XML_FUNCTIONS}, nil + return []string{XML_FUNCTIONS_NAME}, nil } } return nil, nil diff --git a/yb-voyager/src/queryissue/unsupported_ddls.go b/yb-voyager/src/query/queryissue/detectors_ddl.go similarity index 75% rename from yb-voyager/src/queryissue/unsupported_ddls.go rename to yb-voyager/src/query/queryissue/detectors_ddl.go index fe4121e600..f2b6b2f9ff 100644 --- a/yb-voyager/src/queryissue/unsupported_ddls.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -22,15 +22,14 @@ import ( "github.com/samber/lo" - "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) // DDLIssueDetector interface defines methods for detecting issues in DDL objects type DDLIssueDetector interface { - DetectIssues(queryparser.DDLObject) ([]issue.IssueInstance, error) + DetectIssues(queryparser.DDLObject) ([]QueryIssue, error) } //=============TABLE ISSUE DETECTOR =========================== @@ -40,18 +39,18 @@ type TableIssueDetector struct { ParserIssueDetector } -func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { table, ok := obj.(*queryparser.Table) if !ok { return nil, fmt.Errorf("invalid object type: expected Table") } - var issues []issue.IssueInstance + var issues []QueryIssue // Check for generated columns if len(table.GeneratedColumns) > 0 { - issues = append(issues, issue.NewGeneratedColumnsIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewGeneratedColumnsIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", // query string table.GeneratedColumns, @@ -60,16 +59,16 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is // Check for unlogged table if table.IsUnlogged { - issues = append(issues, issue.NewUnloggedTableIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewUnloggedTableIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", // query string )) } if table.IsInherited { - issues = append(issues, issue.NewInheritanceIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewInheritanceIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", )) @@ -79,16 +78,16 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is for _, c := range table.Constraints { if c.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { - issues = append(issues, issue.NewExclusionConstraintIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewExclusionConstraintIssue( + TABLE_OBJECT_TYPE, fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), "", )) } if c.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && c.IsDeferrable { - issues = append(issues, issue.NewDeferrableConstraintIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewDeferrableConstraintIssue( + TABLE_OBJECT_TYPE, fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), "", )) @@ -105,8 +104,8 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is if !ok { continue } - issues = append(issues, issue.NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( + TABLE_OBJECT_TYPE, fmt.Sprintf("%s, constraint: %s", table.GetObjectName(), c.ConstraintName), "", typeName, @@ -130,10 +129,10 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is isUnsupportedDatatypeInLiveWithFFOrFB := isUnsupportedDatatypeInLiveWithFFOrFBList || isUDTDatatype || isArrayOfEnumsDatatype if isUnsupportedDatatype { - reportUnsupportedDatatypes(col, issue.TABLE_OBJECT_TYPE, table.GetObjectName(), &issues) + reportUnsupportedDatatypes(col, TABLE_OBJECT_TYPE, table.GetObjectName(), &issues) } else if isUnsupportedDatatypeInLive { - issues = append(issues, issue.NewUnsupportedDatatypesForLMIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewUnsupportedDatatypesForLMIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", col.TypeName, @@ -145,8 +144,8 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is if col.IsArrayType { // For Array cases to make it clear in issue reportTypeName = fmt.Sprintf("%s[]", reportTypeName) } - issues = append(issues, issue.NewUnsupportedDatatypesForLMWithFFOrFBIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewUnsupportedDatatypesForLMWithFFOrFBIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", reportTypeName, @@ -165,8 +164,8 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is */ alterAddPk := d.primaryConsInAlter[table.GetObjectName()] if alterAddPk != nil { - issues = append(issues, issue.NewAlterTableAddPKOnPartiionIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewAlterTableAddPKOnPartiionIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), alterAddPk.Query, )) @@ -175,8 +174,8 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is uniqueKeyColumns := table.UniqueKeyColumns() if table.IsExpressionPartition && (len(primaryKeyColumns) > 0 || len(uniqueKeyColumns) > 0) { - issues = append(issues, issue.NewExpressionPartitionIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewExpressionPartitionIssue( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", )) @@ -184,16 +183,16 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is if table.PartitionStrategy == queryparser.LIST_PARTITION && len(table.PartitionColumns) > 1 { - issues = append(issues, issue.NewMultiColumnListPartition( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewMultiColumnListPartition( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", )) } partitionColumnsNotInPK, _ := lo.Difference(table.PartitionColumns, primaryKeyColumns) if len(primaryKeyColumns) > 0 && len(partitionColumnsNotInPK) > 0 { - issues = append(issues, issue.NewInsufficientColumnInPKForPartition( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewInsufficientColumnInPKForPartition( + TABLE_OBJECT_TYPE, table.GetObjectName(), "", partitionColumnsNotInPK, @@ -204,24 +203,24 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is return issues, nil } -func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, objName string, issues *[]issue.IssueInstance) { +func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, objName string, issues *[]QueryIssue) { switch col.TypeName { case "xml": - *issues = append(*issues, issue.NewXMLDatatypeIssue( + *issues = append(*issues, NewXMLDatatypeIssue( objType, objName, "", col.ColumnName, )) case "xid": - *issues = append(*issues, issue.NewXIDDatatypeIssue( + *issues = append(*issues, NewXIDDatatypeIssue( objType, objName, "", col.ColumnName, )) case "geometry", "geography", "box2d", "box3d", "topogeometry": - *issues = append(*issues, issue.NewPostGisDatatypeIssue( + *issues = append(*issues, NewPostGisDatatypeIssue( objType, objName, "", @@ -229,7 +228,7 @@ func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, obj col.ColumnName, )) default: - *issues = append(*issues, issue.NewUnsupportedDatatypesIssue( + *issues = append(*issues, NewUnsupportedDatatypesIssue( objType, objName, "", @@ -245,15 +244,15 @@ func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, obj type ForeignTableIssueDetector struct{} -func (f *ForeignTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (f *ForeignTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { foreignTable, ok := obj.(*queryparser.ForeignTable) if !ok { return nil, fmt.Errorf("invalid object type: expected Foreign Table") } - issues := make([]issue.IssueInstance, 0) + issues := make([]QueryIssue, 0) - issues = append(issues, issue.NewForeignTableIssue( - issue.FOREIGN_TABLE_OBJECT_TYPE, + issues = append(issues, NewForeignTableIssue( + FOREIGN_TABLE_OBJECT_TYPE, foreignTable.GetObjectName(), "", foreignTable.ServerName, @@ -262,7 +261,7 @@ func (f *ForeignTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]i for _, col := range foreignTable.Columns { isUnsupportedDatatype := utils.ContainsAnyStringFromSlice(srcdb.PostgresUnsupportedDataTypes, col.TypeName) if isUnsupportedDatatype { - reportUnsupportedDatatypes(col, issue.FOREIGN_TABLE_OBJECT_TYPE, foreignTable.GetObjectName(), &issues) + reportUnsupportedDatatypes(col, FOREIGN_TABLE_OBJECT_TYPE, foreignTable.GetObjectName(), &issues) } } @@ -277,18 +276,18 @@ type IndexIssueDetector struct { ParserIssueDetector } -func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { index, ok := obj.(*queryparser.Index) if !ok { return nil, fmt.Errorf("invalid object type: expected Index") } - var issues []issue.IssueInstance + var issues []QueryIssue // Check for unsupported index methods if slices.Contains(UnsupportedIndexMethods, index.AccessMethod) { - issues = append(issues, issue.NewUnsupportedIndexMethodIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewUnsupportedIndexMethodIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", // query string index.AccessMethod, @@ -297,8 +296,8 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is // Check for storage parameters if index.NumStorageOptions > 0 { - issues = append(issues, issue.NewStorageParameterIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewStorageParameterIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", // query string )) @@ -307,8 +306,8 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is //GinVariations if index.AccessMethod == GIN_ACCESS_METHOD { if len(index.Params) > 1 { - issues = append(issues, issue.NewMultiColumnGinIndexIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewMultiColumnGinIndexIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", )) @@ -316,8 +315,8 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is //In case only one Param is there param := index.Params[0] if param.SortByOrder != queryparser.DEFAULT_SORTING_ORDER { - issues = append(issues, issue.NewOrderedGinIndexIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewOrderedGinIndexIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", )) @@ -341,8 +340,8 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is isUnsupportedType := slices.Contains(UnsupportedIndexDatatypes, param.ExprCastTypeName) isUDTType := slices.Contains(d.compositeTypes, param.GetFullExprCastTypeName()) if param.IsExprCastArrayType { - issues = append(issues, issue.NewIndexOnComplexDatatypesIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewIndexOnComplexDatatypesIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", "array", @@ -352,8 +351,8 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is if isUDTType { reportTypeName = "user_defined_type" } - issues = append(issues, issue.NewIndexOnComplexDatatypesIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewIndexOnComplexDatatypesIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", reportTypeName, @@ -366,8 +365,8 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is if !ok { continue } - issues = append(issues, issue.NewIndexOnComplexDatatypesIssue( - issue.INDEX_OBJECT_TYPE, + issues = append(issues, NewIndexOnComplexDatatypesIssue( + INDEX_OBJECT_TYPE, index.GetObjectName(), "", typeName, @@ -386,41 +385,41 @@ type AlterTableIssueDetector struct { ParserIssueDetector } -func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { alter, ok := obj.(*queryparser.AlterTable) if !ok { return nil, fmt.Errorf("invalid object type: expected AlterTable") } - var issues []issue.IssueInstance + var issues []QueryIssue switch alter.AlterType { case queryparser.SET_OPTIONS: if alter.NumSetAttributes > 0 { - issues = append(issues, issue.NewSetAttributeIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewSetAttributeIssue( + TABLE_OBJECT_TYPE, alter.GetObjectName(), "", // query string )) } case queryparser.ADD_CONSTRAINT: if alter.NumStorageOptions > 0 { - issues = append(issues, issue.NewStorageParameterIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewStorageParameterIssue( + TABLE_OBJECT_TYPE, alter.GetObjectName(), "", // query string )) } if alter.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { - issues = append(issues, issue.NewExclusionConstraintIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewExclusionConstraintIssue( + TABLE_OBJECT_TYPE, fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), "", )) } if alter.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && alter.IsDeferrable { - issues = append(issues, issue.NewDeferrableConstraintIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewDeferrableConstraintIssue( + TABLE_OBJECT_TYPE, fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), "", )) @@ -428,8 +427,8 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]i if alter.ConstraintType == queryparser.PRIMARY_CONSTR_TYPE && aid.partitionTablesMap[alter.GetObjectName()] { - issues = append(issues, issue.NewAlterTableAddPKOnPartiionIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewAlterTableAddPKOnPartiionIssue( + TABLE_OBJECT_TYPE, alter.GetObjectName(), "", )) @@ -446,8 +445,8 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]i if !ok { continue } - issues = append(issues, issue.NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( + TABLE_OBJECT_TYPE, fmt.Sprintf("%s, constraint: %s", alter.GetObjectName(), alter.ConstraintName), "", typeName, @@ -457,15 +456,15 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]i } case queryparser.DISABLE_RULE: - issues = append(issues, issue.NewDisableRuleIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewDisableRuleIssue( + TABLE_OBJECT_TYPE, alter.GetObjectName(), "", // query string alter.RuleName, )) case queryparser.CLUSTER_ON: - issues = append(issues, issue.NewClusterONIssue( - issue.TABLE_OBJECT_TYPE, + issues = append(issues, NewClusterONIssue( + TABLE_OBJECT_TYPE, alter.GetObjectName(), "", // query string )) @@ -479,15 +478,15 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]i // PolicyIssueDetector handles detection of Create policy issues type PolicyIssueDetector struct{} -func (p *PolicyIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (p *PolicyIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { policy, ok := obj.(*queryparser.Policy) if !ok { return nil, fmt.Errorf("invalid object type: expected Policy") } - issues := make([]issue.IssueInstance, 0) + issues := make([]QueryIssue, 0) if len(policy.RoleNames) > 0 { - issues = append(issues, issue.NewPolicyRoleIssue( - issue.POLICY_OBJECT_TYPE, + issues = append(issues, NewPolicyRoleIssue( + POLICY_OBJECT_TYPE, policy.GetObjectName(), "", policy.RoleNames, @@ -503,32 +502,32 @@ type TriggerIssueDetector struct { ParserIssueDetector } -func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { trigger, ok := obj.(*queryparser.Trigger) if !ok { return nil, fmt.Errorf("invalid object type: expected Trigger") } - issues := make([]issue.IssueInstance, 0) + issues := make([]QueryIssue, 0) if trigger.IsConstraint { - issues = append(issues, issue.NewConstraintTriggerIssue( - issue.TRIGGER_OBJECT_TYPE, + issues = append(issues, NewConstraintTriggerIssue( + TRIGGER_OBJECT_TYPE, trigger.GetObjectName(), "", )) } if trigger.NumTransitionRelations > 0 { - issues = append(issues, issue.NewReferencingClauseTrigIssue( - issue.TRIGGER_OBJECT_TYPE, + issues = append(issues, NewReferencingClauseTrigIssue( + TRIGGER_OBJECT_TYPE, trigger.GetObjectName(), "", )) } if trigger.IsBeforeRowTrigger() && tid.partitionTablesMap[trigger.GetTableName()] { - issues = append(issues, issue.NewBeforeRowOnPartitionTableIssue( - issue.TRIGGER_OBJECT_TYPE, + issues = append(issues, NewBeforeRowOnPartitionTableIssue( + TRIGGER_OBJECT_TYPE, trigger.GetObjectName(), "", )) @@ -541,7 +540,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issu type ViewIssueDetector struct{} -func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { return nil, nil } @@ -549,7 +548,7 @@ func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Iss type MViewIssueDetector struct{} -func (v *MViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (v *MViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { return nil, nil } @@ -558,7 +557,7 @@ func (v *MViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.Is // Need to handle all the cases for which we don't have any issues detector type NoOpIssueDetector struct{} -func (n *NoOpIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]issue.IssueInstance, error) { +func (n *NoOpIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { return nil, nil } @@ -584,7 +583,7 @@ func (p *ParserIssueDetector) GetDDLDetector(obj queryparser.DDLObject) (DDLIssu }, nil case *queryparser.ForeignTable: return &ForeignTableIssueDetector{}, nil - case *queryparser.View: + case *queryparser.View: return &ViewIssueDetector{}, nil case *queryparser.MView: return &MViewIssueDetector{}, nil diff --git a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go b/yb-voyager/src/query/queryissue/detectors_test.go similarity index 96% rename from yb-voyager/src/queryissue/unsupported_dml_constructs_test.go rename to yb-voyager/src/query/queryissue/detectors_test.go index c2e0bc3724..194aff8e4e 100644 --- a/yb-voyager/src/queryissue/unsupported_dml_constructs_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) func TestFuncCallDetector(t *testing.T) { @@ -96,7 +96,7 @@ func TestFuncCallDetector(t *testing.T) { parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.Contains(t, unsupportedConstructs, ADVISORY_LOCKS, "Advisory Locks not detected in SQL: %s", sql) + assert.Contains(t, unsupportedConstructs, ADVISORY_LOCKS_NAME, "Advisory Locks not detected in SQL: %s", sql) } } @@ -161,7 +161,7 @@ func TestColumnRefDetector(t *testing.T) { parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.Contains(t, unsupportedConstructs, SYSTEM_COLUMNS, "System Columns not detected in SQL: %s", sql) + assert.Contains(t, unsupportedConstructs, SYSTEM_COLUMNS_NAME, "System Columns not detected in SQL: %s", sql) } } @@ -345,7 +345,7 @@ func TestRangeTableFuncDetector(t *testing.T) { parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS, "XML Functions not detected in SQL: %s", sql) + assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS_NAME, "XML Functions not detected in SQL: %s", sql) } } @@ -495,7 +495,7 @@ WHERE id = 1;`, err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) // The detector should detect XML Functions in these queries - assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS, "XML Functions not detected in SQL: %s", sql) + assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS_NAME, "XML Functions not detected in SQL: %s", sql) } } @@ -530,7 +530,7 @@ RETURNING id, xmlattributes(id AS "ID"), xmlforest(name AS "Name", salary AS "NewSalary", xmin AS "TransactionStartID", xmax AS "TransactionEndID"));`, } - expectedConstructs := []string{ADVISORY_LOCKS, SYSTEM_COLUMNS, XML_FUNCTIONS} + expectedConstructs := []string{ADVISORY_LOCKS_NAME, SYSTEM_COLUMNS_NAME, XML_FUNCTIONS_NAME} detectors := []UnsupportedConstructDetector{ NewFuncCallDetector(), @@ -613,7 +613,7 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { }, } - expectedConstructs := []string{ADVISORY_LOCKS, SYSTEM_COLUMNS, XML_FUNCTIONS} + expectedConstructs := []string{ADVISORY_LOCKS_NAME, SYSTEM_COLUMNS_NAME, XML_FUNCTIONS_NAME} detectors := []UnsupportedConstructDetector{ NewFuncCallDetector(), diff --git a/yb-voyager/src/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go similarity index 100% rename from yb-voyager/src/queryissue/helpers.go rename to yb-voyager/src/query/queryissue/helpers.go diff --git a/yb-voyager/src/issue/ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go similarity index 56% rename from yb-voyager/src/issue/ddl.go rename to yb-voyager/src/query/queryissue/issues_ddl.go index 74c5573f89..48bf327e57 100644 --- a/yb-voyager/src/issue/ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -14,124 +14,119 @@ See the License for the specific language governing permissions and limitations under the License. */ -package issue +package queryissue import ( "fmt" "strings" -) -const ( - DOCS_LINK_PREFIX = "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/" - POSTGRESQL_PREFIX = "postgresql/" - MYSQL_PREFIX = "mysql/" - ORACLE_PREFIX = "oracle/" + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" ) -var generatedColumnsIssue = Issue{ +var generatedColumnsIssue = issue.Issue{ Type: STORED_GENERATED_COLUMNS, TypeName: "Stored generated columns are not supported.", GH: "https://github.com/yugabyte/yugabyte-db/issues/10695", Suggestion: "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#generated-always-as-stored-type-column-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", } -func NewGeneratedColumnsIssue(objectType string, objectName string, sqlStatement string, generatedColumns []string) IssueInstance { +func NewGeneratedColumnsIssue(objectType string, objectName string, sqlStatement string, generatedColumns []string) QueryIssue { issue := generatedColumnsIssue issue.TypeName = issue.TypeName + fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")) - return newIssueInstance(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var unloggedTableIssue = Issue{ +var unloggedTableIssue = issue.Issue{ Type: UNLOGGED_TABLE, TypeName: "UNLOGGED tables are not supported yet.", GH: "https://github.com/yugabyte/yugabyte-db/issues/1129/", Suggestion: "Remove UNLOGGED keyword to make it work", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unlogged-table-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", } -func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement string) IssueInstance { +func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} //for UNLOGGED TABLE as its not reported in the TABLE objects if objectType == "TABLE" { details["INCREASE_INVALID_COUNT"] = false } - return newIssueInstance(unloggedTableIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(unloggedTableIssue, objectType, objectName, sqlStatement, details) } -var unsupportedIndexMethodIssue = Issue{ +var unsupportedIndexMethodIssue = issue.Issue{ Type: UNSUPPORTED_INDEX_METHOD, TypeName: "Schema contains %s index which is not supported.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1337", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#gist-brin-and-spgist-index-types-are-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", } -func NewUnsupportedIndexMethodIssue(objectType string, objectName string, sqlStatement string, indexAccessMethod string) IssueInstance { +func NewUnsupportedIndexMethodIssue(objectType string, objectName string, sqlStatement string, indexAccessMethod string) QueryIssue { issue := unsupportedIndexMethodIssue issue.TypeName = fmt.Sprintf(unsupportedIndexMethodIssue.TypeName, strings.ToUpper(indexAccessMethod)) - return newIssueInstance(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var storageParameterIssue = Issue{ +var storageParameterIssue = issue.Issue{ Type: STORAGE_PARAMETER, TypeName: "Storage parameters are not supported yet.", GH: "https://github.com/yugabyte/yugabyte-db/issues/23467", Suggestion: "Remove the storage parameters from the DDL", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", } -func NewStorageParameterIssue(objectType string, objectName string, sqlStatement string) IssueInstance { +func NewStorageParameterIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} //for ALTER AND INDEX both same struct now how to differentiate which one to not if objectType == "TABLE" { details["INCREASE_INVALID_COUNT"] = false } - return newIssueInstance(storageParameterIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(storageParameterIssue, objectType, objectName, sqlStatement, details) } -var setAttributeIssue = Issue{ +var setAttributeIssue = issue.Issue{ Type: SET_ATTRIBUTES, TypeName: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-alter-table-ddl-variants-in-source-schema", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", } -func NewSetAttributeIssue(objectType string, objectName string, sqlStatement string) IssueInstance { +func NewSetAttributeIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} //for ALTER AND INDEX both same struct now how to differentiate which one to not if objectType == "TABLE" { details["INCREASE_INVALID_COUNT"] = false } - return newIssueInstance(setAttributeIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(setAttributeIssue, objectType, objectName, sqlStatement, details) } -var clusterOnIssue = Issue{ +var clusterOnIssue = issue.Issue{ Type: CLUSTER_ON, TypeName: "ALTER TABLE CLUSTER not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema.", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-alter-table-ddl-variants-in-source-schema", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", } -func NewClusterONIssue(objectType string, objectName string, sqlStatement string) IssueInstance { +func NewClusterONIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} //for ALTER AND INDEX both same struct now how to differentiate which one to not if objectType == "TABLE" { details["INCREASE_INVALID_COUNT"] = false } - return newIssueInstance(clusterOnIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(clusterOnIssue, objectType, objectName, sqlStatement, details) } -var disableRuleIssue = Issue{ +var disableRuleIssue = issue.Issue{ Type: DISABLE_RULE, TypeName: "ALTER TABLE name DISABLE RULE not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove this and the rule '%s' from the exported schema to be not enabled on the table.", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-alter-table-ddl-variants-in-source-schema", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", } -func NewDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) IssueInstance { +func NewDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) QueryIssue { details := map[string]interface{}{} //for ALTER AND INDEX both same struct now how to differentiate which one to not if objectType == "TABLE" { @@ -139,251 +134,251 @@ func NewDisableRuleIssue(objectType string, objectName string, sqlStatement stri } issue := disableRuleIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, ruleName) - return newIssueInstance(issue, objectType, objectName, sqlStatement, details) + return newQueryIssue(issue, objectType, objectName, sqlStatement, details) } -var exclusionConstraintIssue = Issue{ +var exclusionConstraintIssue = issue.Issue{ Type: EXCLUSION_CONSTRAINTS, TypeName: "Exclusion constraint is not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/3944", Suggestion: "Refer docs link for details on possible workaround", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#exclusion-constraints-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", } -func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(exclusionConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(exclusionConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var deferrableConstraintIssue = Issue{ +var deferrableConstraintIssue = issue.Issue{ Type: DEFERRABLE_CONSTRAINTS, TypeName: "DEFERRABLE constraints not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1709", Suggestion: "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", } -func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(deferrableConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(deferrableConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var multiColumnGinIndexIssue = Issue{ +var multiColumnGinIndexIssue = issue.Issue{ Type: MULTI_COLUMN_GIN_INDEX, TypeName: "Schema contains gin index on multi column which is not supported.", GH: "https://github.com/yugabyte/yugabyte-db/issues/10652", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#gin-indexes-on-multiple-columns-are-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gin-indexes-on-multiple-columns-are-not-supported", } -func NewMultiColumnGinIndexIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(multiColumnGinIndexIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewMultiColumnGinIndexIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(multiColumnGinIndexIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var orderedGinIndexIssue = Issue{ +var orderedGinIndexIssue = issue.Issue{ Type: ORDERED_GIN_INDEX, TypeName: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", GH: "https://github.com/yugabyte/yugabyte-db/issues/10653", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#issue-in-some-unsupported-cases-of-gin-indexes", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#issue-in-some-unsupported-cases-of-gin-indexes", } -func NewOrderedGinIndexIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(orderedGinIndexIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewOrderedGinIndexIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(orderedGinIndexIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var policyRoleIssue = Issue{ +var policyRoleIssue = issue.Issue{ Type: POLICY_WITH_ROLES, TypeName: "Policy require roles to be created.", Suggestion: "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", GH: "https://github.com/yugabyte/yb-voyager/issues/1655", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#policies-on-users-in-source-require-manual-user-creation", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", } -func NewPolicyRoleIssue(objectType string, objectName string, SqlStatement string, roles []string) IssueInstance { +func NewPolicyRoleIssue(objectType string, objectName string, SqlStatement string, roles []string) QueryIssue { issue := policyRoleIssue issue.TypeName = fmt.Sprintf("%s Users - (%s)", issue.TypeName, strings.Join(roles, ",")) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var constraintTriggerIssue = Issue{ +var constraintTriggerIssue = issue.Issue{ Type: CONSTRAINT_TRIGGER, TypeName: "CONSTRAINT TRIGGER not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1709", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#constraint-trigger-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", } -func NewConstraintTriggerIssue(objectType string, objectName string, SqlStatement string) IssueInstance { +func NewConstraintTriggerIssue(objectType string, objectName string, SqlStatement string) QueryIssue { details := map[string]interface{}{} //for CONSTRAINT TRIGGER we don't have separate object type TODO: fix if objectType == "TRIGGER" { details["INCREASE_INVALID_COUNT"] = false } - return newIssueInstance(constraintTriggerIssue, objectType, objectName, SqlStatement, details) + return newQueryIssue(constraintTriggerIssue, objectType, objectName, SqlStatement, details) } -var referencingClauseInTriggerIssue = Issue{ +var referencingClauseInTriggerIssue = issue.Issue{ Type: REFERENCING_CLAUSE_FOR_TRIGGERS, TypeName: "REFERENCING clause (transition tables) not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1668", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#referencing-clause-for-triggers", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", } -func NewReferencingClauseTrigIssue(objectType string, objectName string, SqlStatement string) IssueInstance { - return newIssueInstance(referencingClauseInTriggerIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewReferencingClauseTrigIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(referencingClauseInTriggerIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var beforeRowTriggerOnPartitionTableIssue = Issue{ +var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ Type: BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, TypeName: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#before-row-triggers-on-partitioned-tables", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", GH: "https://github.com/yugabyte/yugabyte-db/issues/24830", Suggestion: "Create the triggers on individual partitions.", } -func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, SqlStatement string) IssueInstance { - return newIssueInstance(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var alterTableAddPKOnPartitionIssue = Issue{ +var alterTableAddPKOnPartitionIssue = issue.Issue{ Type: ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, TypeName: "Adding primary key to a partitioned table is not supported yet.", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#adding-primary-key-to-a-partitioned-table-results-in-an-error", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", GH: "https://github.com/yugabyte/yugabyte-db/issues/10074", } -func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, SqlStatement string) IssueInstance { +func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { details := map[string]interface{}{} //for ALTER AND INDEX both same struct now how to differentiate which one to not if objectType == "TABLE" { details["INCREASE_INVALID_COUNT"] = false } - return newIssueInstance(alterTableAddPKOnPartitionIssue, objectType, objectName, SqlStatement, details) + return newQueryIssue(alterTableAddPKOnPartitionIssue, objectType, objectName, SqlStatement, details) } -var expressionPartitionIssue = Issue{ +var expressionPartitionIssue = issue.Issue{ Type: EXPRESSION_PARTITION_WITH_PK_UK, TypeName: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", Suggestion: "Remove the Constriant from the table definition", GH: "https://github.com/yugabyte/yb-voyager/issues/698", - DocsLink: DOCS_LINK_PREFIX + MYSQL_PREFIX + "#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", } -func NewExpressionPartitionIssue(objectType string, objectName string, SqlStatement string) IssueInstance { - return newIssueInstance(expressionPartitionIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewExpressionPartitionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(expressionPartitionIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var multiColumnListPartition = Issue{ +var multiColumnListPartition = issue.Issue{ Type: MULTI_COLUMN_LIST_PARTITION, TypeName: `cannot use "list" partition strategy with more than one column`, Suggestion: "Make it a single column partition by list or choose other supported Partitioning methods", GH: "https://github.com/yugabyte/yb-voyager/issues/699", - DocsLink: DOCS_LINK_PREFIX + MYSQL_PREFIX + "#multi-column-partition-by-list-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", } -func NewMultiColumnListPartition(objectType string, objectName string, SqlStatement string) IssueInstance { - return newIssueInstance(multiColumnListPartition, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewMultiColumnListPartition(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(multiColumnListPartition, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var insufficientColumnsInPKForPartition = Issue{ +var insufficientColumnsInPKForPartition = issue.Issue{ Type: INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, TypeName: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", Suggestion: "Add all Partition columns to Primary Key", GH: "https://github.com/yugabyte/yb-voyager/issues/578", - DocsLink: DOCS_LINK_PREFIX + ORACLE_PREFIX + "#partition-key-column-not-part-of-primary-key-columns", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", } -func NewInsufficientColumnInPKForPartition(objectType string, objectName string, SqlStatement string, partitionColumnsNotInPK []string) IssueInstance { +func NewInsufficientColumnInPKForPartition(objectType string, objectName string, SqlStatement string, partitionColumnsNotInPK []string) QueryIssue { issue := insufficientColumnsInPKForPartition issue.TypeName = fmt.Sprintf("%s - (%s)", issue.TypeName, strings.Join(partitionColumnsNotInPK, ", ")) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var xmlDatatypeIssue = Issue{ +var xmlDatatypeIssue = issue.Issue{ Type: XML_DATATYPE, TypeName: "Unsupported datatype - xml", Suggestion: "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", GH: "https://github.com/yugabyte/yugabyte-db/issues/1043", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#data-ingestion-on-xml-data-type-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", } -func NewXMLDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) IssueInstance { +func NewXMLDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { issue := xmlDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var xidDatatypeIssue = Issue{ +var xidDatatypeIssue = issue.Issue{ Type: XID_DATATYPE, TypeName: "Unsupported datatype - xid", Suggestion: "Functions for this type e.g. txid_current are not supported in YugabyteDB yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/15638", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#xid-functions-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", } -func NewXIDDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) IssueInstance { +func NewXIDDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { issue := xidDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var postgisDatatypeIssue = Issue{ +var postgisDatatypeIssue = issue.Issue{ Type: POSTGIS_DATATYPES, TypeName: "Unsupported datatype", GH: "https://github.com/yugabyte/yugabyte-db/issues/11323", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-yugabytedb", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } -func NewPostGisDatatypeIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { +func NewPostGisDatatypeIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { issue := postgisDatatypeIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var unsupportedDatatypesIssue = Issue{ +var unsupportedDatatypesIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES, TypeName: "Unsupported datatype", GH: "https://github.com/yugabyte/yb-voyager/issues/1731", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-yugabytedb", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } -func NewUnsupportedDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { +func NewUnsupportedDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var unsupportedDatatypesForLiveMigrationIssue = Issue{ +var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION, TypeName: "Unsupported datatype for Live migration", GH: "https://github.com/yugabyte/yb-voyager/issues/1731", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-voyager-during-live-migration", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } -func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { +func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = Issue{ +var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB, TypeName: "Unsupported datatype for Live migration with fall-forward/fallback", GH: "https://github.com/yugabyte/yb-voyager/issues/1731", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#unsupported-datatypes-by-voyager-during-live-migration", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } -func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) IssueInstance { +func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationWithFFOrFBIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var primaryOrUniqueOnUnsupportedIndexTypesIssue = Issue{ +var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ Type: PK_UK_ON_COMPLEX_DATATYPE, TypeName: "Primary key and Unique constraint on column '%s' not yet supported", GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, } -func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, SqlStatement string, typeName string, increaseInvalidCnt bool) IssueInstance { +func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, SqlStatement string, typeName string, increaseInvalidCnt bool) QueryIssue { details := map[string]interface{}{} //for ALTER not increasing count, but for Create increasing TODO: fix if !increaseInvalidCnt { @@ -391,57 +386,57 @@ func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objec } issue := primaryOrUniqueOnUnsupportedIndexTypesIssue issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, details) + return newQueryIssue(issue, objectType, objectName, SqlStatement, details) } -var indexOnComplexDatatypesIssue = Issue{ +var indexOnComplexDatatypesIssue = issue.Issue{ Type: INDEX_ON_COMPLEX_DATATYPE, TypeName: "INDEX on column '%s' not yet supported", GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#indexes-on-some-complex-data-types-are-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", } -func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string) IssueInstance { +func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string) QueryIssue { issue := indexOnComplexDatatypesIssue issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var foreignTableIssue = Issue{ +var foreignTableIssue = issue.Issue{ Type: FOREIGN_TABLE, TypeName: "Foreign tables require manual intervention.", GH: "https://github.com/yugabyte/yb-voyager/issues/1627", Suggestion: "SERVER '%s', and USER MAPPING should be created manually on the target to create and use the foreign table", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#foreign-table-in-the-source-database-requires-server-and-user-mapping", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", } -func NewForeignTableIssue(objectType string, objectName string, SqlStatement string, serverName string) IssueInstance { +func NewForeignTableIssue(objectType string, objectName string, SqlStatement string, serverName string) QueryIssue { issue := foreignTableIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, serverName) - return newIssueInstance(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } -var inheritanceIssue = Issue{ +var inheritanceIssue = issue.Issue{ Type: INHERITANCE, TypeName: "TABLE INHERITANCE not supported in YugabyteDB", GH: "https://github.com/YugaByte/yugabyte-db/issues/1129", - DocsLink: DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#table-inheritance-is-not-supported", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", } -func NewInheritanceIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(inheritanceIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewInheritanceIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(inheritanceIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var percentTypeSyntax = Issue{ - Type: REFERENCED_TYPE_DECLARATION, - TypeName: "Referenced type declaration of variables", + +var percentTypeSyntax = issue.Issue{ + Type: REFERENCED_TYPE_DECLARATION, + TypeName: "Referenced type declaration of variables", TypeDescription: "", - Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", - GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", + GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", } -func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(percentTypeSyntax, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(percentTypeSyntax, objectType, objectName, sqlStatement, map[string]interface{}{}) } - diff --git a/yb-voyager/src/queryissue/ddl_issues_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go similarity index 100% rename from yb-voyager/src/queryissue/ddl_issues_test.go rename to yb-voyager/src/query/queryissue/issues_ddl_test.go diff --git a/yb-voyager/src/issue/dml.go b/yb-voyager/src/query/queryissue/issues_dml.go similarity index 71% rename from yb-voyager/src/issue/dml.go rename to yb-voyager/src/query/queryissue/issues_dml.go index 1479ff9370..c1f21b9f96 100644 --- a/yb-voyager/src/issue/dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package issue +package queryissue -var advisoryLocksIssue = Issue{ +import "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" + +var advisoryLocksIssue = issue.Issue{ Type: ADVISORY_LOCKS, TypeName: "Advisory Locks", TypeDescription: "", @@ -25,11 +27,11 @@ var advisoryLocksIssue = Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", } -func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(advisoryLocksIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(advisoryLocksIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var systemColumnsIssue = Issue{ +var systemColumnsIssue = issue.Issue{ Type: SYSTEM_COLUMNS, TypeName: "System Columns", TypeDescription: "", @@ -38,11 +40,11 @@ var systemColumnsIssue = Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", } -func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(systemColumnsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(systemColumnsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var xmlFunctionsIssue = Issue{ +var xmlFunctionsIssue = issue.Issue{ Type: XML_FUNCTIONS, TypeName: "XML Functions", TypeDescription: "", @@ -51,6 +53,6 @@ var xmlFunctionsIssue = Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", } -func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement string) IssueInstance { - return newIssueInstance(xmlFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(xmlFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } diff --git a/yb-voyager/src/queryissue/queryissue.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go similarity index 87% rename from yb-voyager/src/queryissue/queryissue.go rename to yb-voyager/src/query/queryissue/parser_issue_detector.go index 6f9b31436c..27d28fa936 100644 --- a/yb-voyager/src/queryissue/queryissue.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -25,8 +25,7 @@ import ( log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" - "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" - "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -80,7 +79,7 @@ func (p *ParserIssueDetector) GetEnumTypes() []string { return p.enumTypes } -func (p *ParserIssueDetector) GetAllIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) GetAllIssues(query string, targetDbVersion *ybversion.YBVersion) ([]QueryIssue, error) { issues, err := p.getAllIssues(query) if err != nil { return issues, err @@ -89,7 +88,7 @@ func (p *ParserIssueDetector) GetAllIssues(query string, targetDbVersion *ybvers return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) } -func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) getAllIssues(query string) ([]QueryIssue, error) { plpgsqlIssues, err := p.getPLPGSQLIssues(query) if err != nil { return nil, fmt.Errorf("error getting plpgsql issues: %v", err) @@ -103,11 +102,11 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]issue.IssueInstance, if err != nil { return nil, fmt.Errorf("error getting ddl issues: %v", err) } - return lo.Flatten([][]issue.IssueInstance{plpgsqlIssues, dmlIssues, ddlIssues}), nil + return lo.Flatten([][]QueryIssue{plpgsqlIssues, dmlIssues, ddlIssues}), nil } -func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []issue.IssueInstance, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { - var filteredIssues []issue.IssueInstance +func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []QueryIssue, targetDbVersion *ybversion.YBVersion) ([]QueryIssue, error) { + var filteredIssues []QueryIssue for _, i := range issues { fixed, err := i.IsFixedIn(targetDbVersion) if err != nil { @@ -120,7 +119,7 @@ func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []issue. return filteredIssues, nil } -func (p *ParserIssueDetector) GetAllPLPGSQLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) GetAllPLPGSQLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]QueryIssue, error) { issues, err := p.getPLPGSQLIssues(query) if err != nil { return issues, nil @@ -129,7 +128,7 @@ func (p *ParserIssueDetector) GetAllPLPGSQLIssues(query string, targetDbVersion return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) } -func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]QueryIssue, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) @@ -141,7 +140,7 @@ func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]issue.IssueInsta if err != nil { return nil, fmt.Errorf("error getting all the queries from query: %w", err) } - var issues []issue.IssueInstance + var issues []QueryIssue for _, plpgsqlQuery := range plpgsqlQueries { issuesInQuery, err := p.getAllIssues(plpgsqlQuery) if err != nil { @@ -158,7 +157,7 @@ func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]issue.IssueInsta } issues = append(issues, percentTypeSyntaxIssues...) - return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { + return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object //e.g. replacing the DML_QUERY and "" to FUNCTION and i.ObjectType = objType @@ -178,7 +177,7 @@ func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]issue.IssueInsta return nil, err } - return lo.Map(issues, func(i issue.IssueInstance, _ int) issue.IssueInstance { + return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object //e.g. replacing the DML_QUERY and "" to FUNCTION and i.ObjectType = objType @@ -252,7 +251,7 @@ func (p *ParserIssueDetector) ParseRequiredDDLs(query string) error { return nil } -func (p *ParserIssueDetector) GetDDLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) GetDDLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]QueryIssue, error) { issues, err := p.getDDLIssues(query) if err != nil { return issues, nil @@ -262,7 +261,7 @@ func (p *ParserIssueDetector) GetDDLIssues(query string, targetDbVersion *ybvers } -func (p *ParserIssueDetector) getDDLIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) getDDLIssues(query string) ([]QueryIssue, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing a query: %v", err) @@ -293,7 +292,7 @@ func (p *ParserIssueDetector) getDDLIssues(query string) ([]issue.IssueInstance, return issues, nil } -func (p *ParserIssueDetector) GetPercentTypeSyntaxIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) GetPercentTypeSyntaxIssues(query string) ([]QueryIssue, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing the query-%s: %v", query, err) @@ -314,16 +313,16 @@ func (p *ParserIssueDetector) GetPercentTypeSyntaxIssues(query string) ([]issue. typeNames = append(typeNames, queryparser.GetReturnTypeOfFunc(parseTree)) } typeNames = append(typeNames, queryparser.GetFuncParametersTypeNames(parseTree)...) - var issues []issue.IssueInstance + var issues []QueryIssue for _, typeName := range typeNames { if strings.HasSuffix(typeName, "%TYPE") { - issues = append(issues, issue.NewPercentTypeSyntaxIssue(objType, objName, typeName)) // TODO: confirm + issues = append(issues, NewPercentTypeSyntaxIssue(objType, objName, typeName)) // TODO: confirm } } return issues, nil } -func (p *ParserIssueDetector) GetDMLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) GetDMLIssues(query string, targetDbVersion *ybversion.YBVersion) ([]QueryIssue, error) { issues, err := p.getDMLIssues(query) if err != nil { return issues, err @@ -333,7 +332,7 @@ func (p *ParserIssueDetector) GetDMLIssues(query string, targetDbVersion *ybvers } // TODO: in future when we will DDL issues detection here we need `GetDDLIssues` -func (p *ParserIssueDetector) getDMLIssues(query string) ([]issue.IssueInstance, error) { +func (p *ParserIssueDetector) getDMLIssues(query string) ([]QueryIssue, error) { parseTree, err := queryparser.Parse(query) if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) @@ -346,7 +345,7 @@ func (p *ParserIssueDetector) getDMLIssues(query string) ([]issue.IssueInstance, //Skip all the DDLs coming to this function return nil, nil } - var result []issue.IssueInstance + var result []QueryIssue var unsupportedConstructs []string visited := make(map[protoreflect.Message]bool) detectors := []UnsupportedConstructDetector{ @@ -377,12 +376,12 @@ func (p *ParserIssueDetector) getDMLIssues(query string) ([]issue.IssueInstance, for _, unsupportedConstruct := range unsupportedConstructs { switch unsupportedConstruct { - case ADVISORY_LOCKS: - result = append(result, issue.NewAdvisoryLocksIssue(issue.DML_QUERY_OBJECT_TYPE, "", query)) - case SYSTEM_COLUMNS: - result = append(result, issue.NewSystemColumnsIssue(issue.DML_QUERY_OBJECT_TYPE, "", query)) - case XML_FUNCTIONS: - result = append(result, issue.NewXmlFunctionsIssue(issue.DML_QUERY_OBJECT_TYPE, "", query)) + case ADVISORY_LOCKS_NAME: + result = append(result, NewAdvisoryLocksIssue(DML_QUERY_OBJECT_TYPE, "", query)) + case SYSTEM_COLUMNS_NAME: + result = append(result, NewSystemColumnsIssue(DML_QUERY_OBJECT_TYPE, "", query)) + case XML_FUNCTIONS_NAME: + result = append(result, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", query)) } } return result, nil diff --git a/yb-voyager/src/query/queryissue/query_issue.go b/yb-voyager/src/query/queryissue/query_issue.go new file mode 100644 index 0000000000..5359879f11 --- /dev/null +++ b/yb-voyager/src/query/queryissue/query_issue.go @@ -0,0 +1,42 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This package has all logic related to detecting issues in queries (DDL or DML). +Entry point is ParserIssueDetector, which makes use queryparser pkg to parse +the query and multiple detectors to figure out issues in the parseTree. +*/ +package queryissue + +import "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" + +type QueryIssue struct { + issue.Issue + ObjectType string // TABLE, FUNCTION, DML_QUERY? + ObjectName string // table name/function name/etc + SqlStatement string + Details map[string]interface{} // additional details about the issue +} + +func newQueryIssue(issue issue.Issue, objectType string, objectName string, sqlStatement string, details map[string]interface{}) QueryIssue { + return QueryIssue{ + Issue: issue, + ObjectType: objectType, + ObjectName: objectName, + SqlStatement: sqlStatement, + Details: details, + } +} diff --git a/yb-voyager/src/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go similarity index 100% rename from yb-voyager/src/queryparser/ddl_processor.go rename to yb-voyager/src/query/queryparser/ddl_processor.go diff --git a/yb-voyager/src/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go similarity index 98% rename from yb-voyager/src/queryparser/helpers_protomsg.go rename to yb-voyager/src/query/queryparser/helpers_protomsg.go index 3403161aa8..88b3a85d64 100644 --- a/yb-voyager/src/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -18,6 +18,7 @@ package queryparser import ( "strings" + pg_query "github.com/pganalyze/pg_query_go/v5" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -30,6 +31,10 @@ const ( XML_FUNCTIONS_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#xml-functions-is-not-yet-supported" ) +func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { + return parseTree.Stmts[0].Stmt.ProtoReflect() +} + func GetMsgFullName(msg protoreflect.Message) string { return string(msg.Descriptor().FullName()) } diff --git a/yb-voyager/src/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go similarity index 98% rename from yb-voyager/src/queryparser/helpers_struct.go rename to yb-voyager/src/query/queryparser/helpers_struct.go index e332547cd7..45d1db3901 100644 --- a/yb-voyager/src/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -21,7 +21,6 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v5" "github.com/samber/lo" - "google.golang.org/protobuf/reflect/protoreflect" ) func DeparseSelectStmt(selectStmt *pg_query.SelectStmt) (string, error) { @@ -43,10 +42,6 @@ func DeparseSelectStmt(selectStmt *pg_query.SelectStmt) (string, error) { return "", nil } -func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { - return parseTree.Stmts[0].Stmt.ProtoReflect() -} - func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) @@ -289,7 +284,6 @@ func GetFuncParametersTypeNames(parseTree *pg_query.ParseResult) []string { return paramTypeNames } - func IsDDL(parseTree *pg_query.ParseResult) (bool, error) { ddlParser, err := GetDDLProcessor(parseTree) if err != nil { @@ -299,4 +293,4 @@ func IsDDL(parseTree *pg_query.ParseResult) (bool, error) { //Considering all the DDLs we have a Processor for as of now. //Not Full-proof as we don't have all DDL types but atleast we will skip all the types we know currently return !ok, nil -} \ No newline at end of file +} diff --git a/yb-voyager/src/queryparser/object_collector.go b/yb-voyager/src/query/queryparser/object_collector.go similarity index 100% rename from yb-voyager/src/queryparser/object_collector.go rename to yb-voyager/src/query/queryparser/object_collector.go diff --git a/yb-voyager/src/queryparser/query_parser.go b/yb-voyager/src/query/queryparser/query_parser.go similarity index 69% rename from yb-voyager/src/queryparser/query_parser.go rename to yb-voyager/src/query/queryparser/query_parser.go index 6e6c4149a1..7246436830 100644 --- a/yb-voyager/src/queryparser/query_parser.go +++ b/yb-voyager/src/query/queryparser/query_parser.go @@ -13,6 +13,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + +/* +This package contains all the logic related to parsing a query string, and extracting details out of it. +We mainly use the pg_query_go library to help with this. + +The main functions in this package are: +1. Use pg_query_go to parse the query string into a ParseResult (i.e. a parseTree) +2. Traverse and process each protobufMessage node of the ParseTree. +3. For PLPGSQL, convert the PLPGSQL to JSON; get all the statements out of the PLPGSQL block. +we can put all the parser related logic (the parsing, the parsing of plpgsql to json, the traversal through the proto messages, the traversal through the nested plpgsql json, adding clauses to statements, etc +*/ package queryparser import ( diff --git a/yb-voyager/src/queryparser/traversal_plpgsql.go b/yb-voyager/src/query/queryparser/traversal_plpgsql.go similarity index 100% rename from yb-voyager/src/queryparser/traversal_plpgsql.go rename to yb-voyager/src/query/queryparser/traversal_plpgsql.go diff --git a/yb-voyager/src/queryparser/traversal.go b/yb-voyager/src/query/queryparser/traversal_proto.go similarity index 100% rename from yb-voyager/src/queryparser/traversal.go rename to yb-voyager/src/query/queryparser/traversal_proto.go diff --git a/yb-voyager/src/queryparser/traversal_test.go b/yb-voyager/src/query/queryparser/traversal_test.go similarity index 100% rename from yb-voyager/src/queryparser/traversal_test.go rename to yb-voyager/src/query/queryparser/traversal_test.go From be98ce8246b32704c4204b08e852ca2a05afea12 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Wed, 11 Dec 2024 07:56:24 +0530 Subject: [PATCH 044/105] Support 2024.2 series (#2058) - Support 2024.2 series and set latest stable to 2024.2.0.0 - Updation of MinVersionFixedIn for various issues will happen in a subsequent PR. --- .github/workflows/issue-tests.yml | 3 +- .../src/query/queryissue/issues_ddl_test.go | 44 ++++++++++++++++++- yb-voyager/src/ybversion/constants.go | 9 +++- yb-voyager/src/ybversion/yb_version.go | 2 +- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 94e95b16fc..52b03f19e4 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -10,8 +10,9 @@ jobs: test-issues-against-all-yb-versions: strategy: + fail-fast: false matrix: - version: [2.23.1.0-b220, 2024.1.3.1-b8, 2.20.8.0-b53, 2.18.9.0-b17] + version: [2.23.1.0-b220, 2024.1.3.1-b8, 2024.2.0.0-b145, 2.20.8.0-b53, 2.18.9.0-b17] env: YB_VERSION: ${{ matrix.version }} YB_CONN_STR: "postgres://yugabyte:yugabyte@127.0.0.1:5433/yugabyte" diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 105f565875..da47f1d141 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" ) @@ -52,6 +53,32 @@ func getConn() (*pgx.Conn, error) { return conn, nil } +func getConnWithNoticeHandler(noticeHandler func(*pgconn.PgConn, *pgconn.Notice)) (*pgx.Conn, error) { + ctx := context.Background() + var connStr string + var err error + if yugabytedbConnStr != "" { + connStr = yugabytedbConnStr + } else { + connStr, err = yugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") + if err != nil { + return nil, err + } + } + + conf, err := pgx.ParseConfig(connStr) + if err != nil { + return nil, err + } + conf.OnNotice = noticeHandler + conn, err := pgx.ConnectConfig(ctx, conf) + if err != nil { + return nil, err + } + + return conn, nil +} + func testXMLFunctionIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -79,13 +106,26 @@ func testStoredGeneratedFunctionsIssue(t *testing.T) { } func testUnloggedTableIssue(t *testing.T) { + noticeFound := false + noticeHandler := func(conn *pgconn.PgConn, notice *pgconn.Notice) { + if notice != nil && notice.Message != "" { + assert.Equal(t, "unlogged option is currently ignored in YugabyteDB, all non-temp tables will be logged", notice.Message) + noticeFound = true + } + } ctx := context.Background() - conn, err := getConn() + conn, err := getConnWithNoticeHandler(noticeHandler) assert.NoError(t, err) defer conn.Close(context.Background()) _, err = conn.Exec(ctx, "CREATE UNLOGGED TABLE unlogged_table (a int)") - assert.ErrorContains(t, err, "UNLOGGED database object not supported yet") + // in 2024.2, UNLOGGED no longer throws an error, just a notice + if noticeFound { + return + } else { + assert.ErrorContains(t, err, "UNLOGGED database object not supported yet") + } + } func TestDDLIssuesInYBVersion(t *testing.T) { diff --git a/yb-voyager/src/ybversion/constants.go b/yb-voyager/src/ybversion/constants.go index 0858c58ce4..62650292c1 100644 --- a/yb-voyager/src/ybversion/constants.go +++ b/yb-voyager/src/ybversion/constants.go @@ -20,6 +20,7 @@ const ( SERIES_2_18 = "2.18" SERIES_2_20 = "2.20" SERIES_2024_1 = "2024.1" + SERIES_2024_2 = "2024.2" SERIES_2_21 = "2.21" SERIES_2_23 = "2.23" ) @@ -27,6 +28,7 @@ const ( var LatestStable *YBVersion var V2024_1_3_1 *YBVersion +var V2024_2_0_0 *YBVersion func init() { var err error @@ -34,5 +36,10 @@ func init() { if err != nil { panic("could not create version 2024.1.3.1") } - LatestStable = V2024_1_3_1 + V2024_2_0_0, err = NewYBVersion("2024.2.0.0") + if err != nil { + panic("could not create version 2024.2.0.0") + } + + LatestStable = V2024_2_0_0 } diff --git a/yb-voyager/src/ybversion/yb_version.go b/yb-voyager/src/ybversion/yb_version.go index f2afdc660c..b8d2264587 100644 --- a/yb-voyager/src/ybversion/yb_version.go +++ b/yb-voyager/src/ybversion/yb_version.go @@ -28,7 +28,7 @@ import ( // Reference - https://docs.yugabyte.com/preview/releases/ybdb-releases/ var supportedYBVersionStableSeriesOld = []string{SERIES_2_14, SERIES_2_18, SERIES_2_20} -var supportedYBVersionStableSeries = []string{SERIES_2024_1} +var supportedYBVersionStableSeries = []string{SERIES_2024_1, SERIES_2024_2} var supportedYBVersionPreviewSeries = []string{SERIES_2_21, SERIES_2_23} var allSupportedYBVersionSeries = lo.Flatten([][]string{supportedYBVersionStableSeries, supportedYBVersionPreviewSeries, supportedYBVersionStableSeriesOld}) From 43a0f242713bbe10f643dc61274439b479d0a8c3 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 11 Dec 2024 11:31:18 +0530 Subject: [PATCH 045/105] Added unit tests for detecting all the issues with GetAllIssues() (#2041) --- .../src/queryissue/unsupported_ddls_test.go | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 yb-voyager/src/queryissue/unsupported_ddls_test.go diff --git a/yb-voyager/src/queryissue/unsupported_ddls_test.go b/yb-voyager/src/queryissue/unsupported_ddls_test.go new file mode 100644 index 0000000000..9b9bfd5d5f --- /dev/null +++ b/yb-voyager/src/queryissue/unsupported_ddls_test.go @@ -0,0 +1,201 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryissue + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" +) + +const ( + stmt1 = `CREATE OR REPLACE FUNCTION list_high_earners(threshold NUMERIC) RETURNS public.emp1.salary%TYPE AS $$ +DECLARE + emp_name employees.name%TYPE; + emp_salary employees.salary%TYPE; +BEGIN + FOR emp_name, emp_salary IN + SELECT name, salary FROM employees WHERE salary > threshold + LOOP + RAISE NOTICE 'Employee: %, Salary: %', emp_name, emp_salary; + END LOOP; + EXECUTE 'ALTER TABLE employees CLUSTER ON idx;'; + PERFORM pg_advisory_unlock(sender_id); + PERFORM pg_advisory_unlock(receiver_id); + + -- Conditional logic + IF balance >= withdrawal THEN + RAISE NOTICE 'Sufficient balance, processing withdrawal.'; + -- Add the amount to the receiver's account + UPDATE accounts SET balance = balance + amount WHERE account_id = receiver; + ELSIF balance > 0 AND balance < withdrawal THEN + RAISE NOTICE 'Insufficient balance, consider reducing the amount.'; + -- Add the amount to the receiver's account + UPDATE accounts SET balance = balance + amount WHERE account_id = receiver; + ELSE + -- Add the amount to the receiver's account + UPDATE accounts SET balance = balance + amount WHERE account_id = receiver; + RAISE NOTICE 'No funds available.'; + END IF; + + SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type; + + SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department); + RETURN emp_salary; + +END; +$$ LANGUAGE plpgsql;` + stmt2 = `CREATE OR REPLACE FUNCTION process_order(orderid orders.id%TYPE) RETURNS VOID AS $$ +DECLARE + lock_acquired BOOLEAN; +BEGIN + lock_acquired := pg_try_advisory_lock(orderid); -- not able to report this as it is an assignment statement TODO: fix when support this + + IF NOT lock_acquired THEN + RAISE EXCEPTION 'Order % already being processed by another session', orderid; + END IF; + + UPDATE orders + SET processed_at = NOW() + WHERE orders.order_id = orderid; + + RAISE NOTICE 'Order % processed successfully', orderid; + + EXECUTE 'ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70)'; + + EXECUTE 'CREATE UNLOGGED TABLE tbl_unlog (id int, val text);'; + + EXECUTE 'CREATE INDEX idx_example ON example_table USING gin(name, name1);'; + + EXECUTE 'CREATE INDEX idx_example ON schema1.example_table USING gist(name);'; + + PERFORM pg_advisory_unlock(orderid); +END; +$$ LANGUAGE plpgsql;` + stmt3 = `CREATE INDEX abc ON public.example USING btree (new_id) WITH (fillfactor='70');` + stmt4 = `ALTER TABLE public.example DISABLE RULE example_rule;` + stmt5 = `ALTER TABLE abc ADD CONSTRAINT cnstr_id UNIQUE (id) DEFERRABLE;` + stmt6 = `SELECT id, first_name FROM employees WHERE pg_try_advisory_lock(600) IS TRUE AND salary > 700;` + stmt7 = `SELECT xmin, COUNT(*) FROM employees GROUP BY xmin HAVING COUNT(*) > 1;` + stmt8 = `SELECT id, xml_column, xpath('/root/element/@attribute', xml_column) as xpath_resuls FROM xml_documents;` + stmt9 = `CREATE TABLE order_details ( + detail_id integer NOT NULL, + quantity integer, + price_per_unit numeric, + amount numeric GENERATED ALWAYS AS (((quantity)::numeric * price_per_unit)) STORED +);` + stmt10 = `CREATE TABLE test_non_pk_multi_column_list ( + id numeric NOT NULL PRIMARY KEY, + country_code varchar(3), + record_type varchar(5), + descriptions varchar(50) +) PARTITION BY LIST (country_code, record_type) ;` + + stmt11 = `CREATE TABLE "Test"( + id int, + room_id int, + time_range trange, + roomid int, + timerange tsrange, + EXCLUDE USING gist (room_id WITH =, time_range WITH &&), + CONSTRAINT no_time_overlap_constr EXCLUDE USING gist (roomid WITH =, timerange WITH &&) +);` + stmt12 = `CREATE TABLE test_dt (id int, d daterange);` + stmt13 = `CREATE INDEX idx_on_daterange on test_dt (d);` +) + +func TestAllDDLIssues(t *testing.T) { + requiredDDLs := []string{stmt12} + parserIssueDetector := NewParserIssueDetector() + stmtsWithExpectedIssues := map[string][]issue.IssueInstance{ + stmt1: []issue.IssueInstance{ + issue.NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "public.emp1.salary%TYPE"), + issue.NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.name%TYPE"), + issue.NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.salary%TYPE"), + issue.NewClusterONIssue("FUNCTION", "list_high_earners", "ALTER TABLE employees CLUSTER ON idx;"), + issue.NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(sender_id);"), + issue.NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(receiver_id);"), + issue.NewXmlFunctionsIssue("FUNCTION", "list_high_earners", "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;"), + issue.NewSystemColumnsIssue("FUNCTION", "list_high_earners", "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);"), + }, + stmt2: []issue.IssueInstance{ + issue.NewPercentTypeSyntaxIssue("FUNCTION", "process_order", "orders.id%TYPE"), + issue.NewStorageParameterIssue("FUNCTION", "process_order", "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70);"), + issue.NewUnloggedTableIssue("FUNCTION", "process_order", "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);"), + issue.NewMultiColumnGinIndexIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON example_table USING gin(name, name1);"), + issue.NewUnsupportedIndexMethodIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON schema1.example_table USING gist(name);", "gist"), + issue.NewAdvisoryLocksIssue("FUNCTION", "process_order", "SELECT pg_advisory_unlock(orderid);"), + }, + stmt3: []issue.IssueInstance{ + issue.NewStorageParameterIssue("INDEX", "abc ON public.example", stmt3), + }, + stmt4: []issue.IssueInstance{ + issue.NewDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), + }, + stmt5: []issue.IssueInstance{ + issue.NewDeferrableConstraintIssue("TABLE", "abc, constraint: (cnstr_id)", stmt5), + }, + stmt6: []issue.IssueInstance{ + issue.NewAdvisoryLocksIssue("DML_QUERY", "", stmt6), + }, + stmt7: []issue.IssueInstance{ + issue.NewSystemColumnsIssue("DML_QUERY", "", stmt7), + }, + stmt8: []issue.IssueInstance{ + issue.NewXmlFunctionsIssue("DML_QUERY", "", stmt8), + }, + stmt9: []issue.IssueInstance{ + issue.NewGeneratedColumnsIssue("TABLE", "order_details", stmt9, []string{"amount"}), + }, + stmt10: []issue.IssueInstance{ + issue.NewMultiColumnListPartition("TABLE", "test_non_pk_multi_column_list", stmt10), + issue.NewInsufficientColumnInPKForPartition("TABLE", "test_non_pk_multi_column_list", stmt10, []string{"country_code", "record_type"}), + }, + stmt11: []issue.IssueInstance{ + issue.NewExclusionConstraintIssue("TABLE", "Test, constraint: (Test_room_id_time_range_excl)", stmt11), + issue.NewExclusionConstraintIssue("TABLE", "Test, constraint: (no_time_overlap_constr)", stmt11), + }, + stmt13: []issue.IssueInstance{ + issue.NewIndexOnComplexDatatypesIssue("INDEX", "idx_on_daterange ON test_dt", stmt13, "daterange"), + }, + } + + for _, stmt := range requiredDDLs { + err := parserIssueDetector.ParseRequiredDDLs(stmt) + assert.NoError(t, err, "Error parsing required ddl: %s", stmt) + } + for stmt, expectedIssues := range stmtsWithExpectedIssues { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(issueInstance issue.IssueInstance) bool { + typeNameMatches := issueInstance.TypeName == expectedIssue.TypeName + queryMatches := issueInstance.SqlStatement == expectedIssue.SqlStatement + objectNameMatches := issueInstance.ObjectName == expectedIssue.ObjectName + objectTypeMatches := issueInstance.ObjectType == expectedIssue.ObjectType + return typeNameMatches && queryMatches && objectNameMatches && objectTypeMatches + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } + +} From 2e1e42ffefe611bcf26807443849dd1f9070fcd9 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 11 Dec 2024 12:56:53 +0530 Subject: [PATCH 046/105] Fixing last commit go build failure as per the changes in the packages query/queryissue (#2066) last commit - https://github.com/yugabyte/yb-voyager/commit/43a0f242713bbe10f643dc61274439b479d0a8c3 --- .../queryissue/unsupported_ddls_test.go | 89 +++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) rename yb-voyager/src/{ => query}/queryissue/unsupported_ddls_test.go (59%) diff --git a/yb-voyager/src/queryissue/unsupported_ddls_test.go b/yb-voyager/src/query/queryissue/unsupported_ddls_test.go similarity index 59% rename from yb-voyager/src/queryissue/unsupported_ddls_test.go rename to yb-voyager/src/query/queryissue/unsupported_ddls_test.go index 9b9bfd5d5f..a04f209846 100644 --- a/yb-voyager/src/queryissue/unsupported_ddls_test.go +++ b/yb-voyager/src/query/queryissue/unsupported_ddls_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -124,56 +123,56 @@ $$ LANGUAGE plpgsql;` func TestAllDDLIssues(t *testing.T) { requiredDDLs := []string{stmt12} parserIssueDetector := NewParserIssueDetector() - stmtsWithExpectedIssues := map[string][]issue.IssueInstance{ - stmt1: []issue.IssueInstance{ - issue.NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "public.emp1.salary%TYPE"), - issue.NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.name%TYPE"), - issue.NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.salary%TYPE"), - issue.NewClusterONIssue("FUNCTION", "list_high_earners", "ALTER TABLE employees CLUSTER ON idx;"), - issue.NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(sender_id);"), - issue.NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(receiver_id);"), - issue.NewXmlFunctionsIssue("FUNCTION", "list_high_earners", "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;"), - issue.NewSystemColumnsIssue("FUNCTION", "list_high_earners", "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);"), + stmtsWithExpectedIssues := map[string][]QueryIssue{ + stmt1: []QueryIssue{ + NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "public.emp1.salary%TYPE"), + NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.name%TYPE"), + NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.salary%TYPE"), + NewClusterONIssue("FUNCTION", "list_high_earners", "ALTER TABLE employees CLUSTER ON idx;"), + NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(sender_id);"), + NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(receiver_id);"), + NewXmlFunctionsIssue("FUNCTION", "list_high_earners", "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;"), + NewSystemColumnsIssue("FUNCTION", "list_high_earners", "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);"), }, - stmt2: []issue.IssueInstance{ - issue.NewPercentTypeSyntaxIssue("FUNCTION", "process_order", "orders.id%TYPE"), - issue.NewStorageParameterIssue("FUNCTION", "process_order", "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70);"), - issue.NewUnloggedTableIssue("FUNCTION", "process_order", "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);"), - issue.NewMultiColumnGinIndexIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON example_table USING gin(name, name1);"), - issue.NewUnsupportedIndexMethodIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON schema1.example_table USING gist(name);", "gist"), - issue.NewAdvisoryLocksIssue("FUNCTION", "process_order", "SELECT pg_advisory_unlock(orderid);"), + stmt2: []QueryIssue{ + NewPercentTypeSyntaxIssue("FUNCTION", "process_order", "orders.id%TYPE"), + NewStorageParameterIssue("FUNCTION", "process_order", "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70);"), + NewUnloggedTableIssue("FUNCTION", "process_order", "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);"), + NewMultiColumnGinIndexIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON example_table USING gin(name, name1);"), + NewUnsupportedIndexMethodIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON schema1.example_table USING gist(name);", "gist"), + NewAdvisoryLocksIssue("FUNCTION", "process_order", "SELECT pg_advisory_unlock(orderid);"), }, - stmt3: []issue.IssueInstance{ - issue.NewStorageParameterIssue("INDEX", "abc ON public.example", stmt3), + stmt3: []QueryIssue{ + NewStorageParameterIssue("INDEX", "abc ON public.example", stmt3), }, - stmt4: []issue.IssueInstance{ - issue.NewDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), + stmt4: []QueryIssue{ + NewDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), }, - stmt5: []issue.IssueInstance{ - issue.NewDeferrableConstraintIssue("TABLE", "abc, constraint: (cnstr_id)", stmt5), + stmt5: []QueryIssue{ + NewDeferrableConstraintIssue("TABLE", "abc, constraint: (cnstr_id)", stmt5), }, - stmt6: []issue.IssueInstance{ - issue.NewAdvisoryLocksIssue("DML_QUERY", "", stmt6), + stmt6: []QueryIssue{ + NewAdvisoryLocksIssue("DML_QUERY", "", stmt6), }, - stmt7: []issue.IssueInstance{ - issue.NewSystemColumnsIssue("DML_QUERY", "", stmt7), + stmt7: []QueryIssue{ + NewSystemColumnsIssue("DML_QUERY", "", stmt7), }, - stmt8: []issue.IssueInstance{ - issue.NewXmlFunctionsIssue("DML_QUERY", "", stmt8), + stmt8: []QueryIssue{ + NewXmlFunctionsIssue("DML_QUERY", "", stmt8), }, - stmt9: []issue.IssueInstance{ - issue.NewGeneratedColumnsIssue("TABLE", "order_details", stmt9, []string{"amount"}), + stmt9: []QueryIssue{ + NewGeneratedColumnsIssue("TABLE", "order_details", stmt9, []string{"amount"}), }, - stmt10: []issue.IssueInstance{ - issue.NewMultiColumnListPartition("TABLE", "test_non_pk_multi_column_list", stmt10), - issue.NewInsufficientColumnInPKForPartition("TABLE", "test_non_pk_multi_column_list", stmt10, []string{"country_code", "record_type"}), + stmt10: []QueryIssue{ + NewMultiColumnListPartition("TABLE", "test_non_pk_multi_column_list", stmt10), + NewInsufficientColumnInPKForPartition("TABLE", "test_non_pk_multi_column_list", stmt10, []string{"country_code", "record_type"}), }, - stmt11: []issue.IssueInstance{ - issue.NewExclusionConstraintIssue("TABLE", "Test, constraint: (Test_room_id_time_range_excl)", stmt11), - issue.NewExclusionConstraintIssue("TABLE", "Test, constraint: (no_time_overlap_constr)", stmt11), + stmt11: []QueryIssue{ + NewExclusionConstraintIssue("TABLE", "Test, constraint: (Test_room_id_time_range_excl)", stmt11), + NewExclusionConstraintIssue("TABLE", "Test, constraint: (no_time_overlap_constr)", stmt11), }, - stmt13: []issue.IssueInstance{ - issue.NewIndexOnComplexDatatypesIssue("INDEX", "idx_on_daterange ON test_dt", stmt13, "daterange"), + stmt13: []QueryIssue{ + NewIndexOnComplexDatatypesIssue("INDEX", "idx_on_daterange ON test_dt", stmt13, "daterange"), }, } @@ -187,11 +186,11 @@ func TestAllDDLIssues(t *testing.T) { assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) for _, expectedIssue := range expectedIssues { - found := slices.ContainsFunc(issues, func(issueInstance issue.IssueInstance) bool { - typeNameMatches := issueInstance.TypeName == expectedIssue.TypeName - queryMatches := issueInstance.SqlStatement == expectedIssue.SqlStatement - objectNameMatches := issueInstance.ObjectName == expectedIssue.ObjectName - objectTypeMatches := issueInstance.ObjectType == expectedIssue.ObjectType + found := slices.ContainsFunc(issues, func(QueryIssue QueryIssue) bool { + typeNameMatches := QueryIssue.TypeName == expectedIssue.TypeName + queryMatches := QueryIssue.SqlStatement == expectedIssue.SqlStatement + objectNameMatches := QueryIssue.ObjectName == expectedIssue.ObjectName + objectTypeMatches := QueryIssue.ObjectType == expectedIssue.ObjectType return typeNameMatches && queryMatches && objectNameMatches && objectTypeMatches }) assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) From 01e18677896553bc137cd1c87736c6be3696cbc2 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 11 Dec 2024 12:57:38 +0530 Subject: [PATCH 047/105] Release notes for 1.8.5, 1.8.6 and 1.8.7 (#2062) --- RELEASE_NOTES.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 11b98494e0..d999d808c1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,74 @@ Included here are the release notes for the [YugabyteDB Voyager](https://docs.yugabyte.com/preview/migrate/) v1 release series. Content will be added as new notable features and changes are available in the patch releases of the YugabyteDB v1 series. +## v1.8.7 - December 10, 2024 + +### New Features + +- Introduced a framework in the `assess-migration` and `analyze-schema` commands to accept the target database version (`--target-db-version` flag) as input and use it for reporting issues not supported in that target version for the source schema. + +### Enhancements + +- Improved permission grant script (`yb-voyager-pg-grant-migration-permissions.sql`) by internally detecting table owners, eliminating the need to specify the `original_owner_of_tables` flag. +- Enhanced reporting of **Unsupported Query Constructs** in the `assess-migration` command by filtering queries to include only those that match user-specified schemas, provided schema information is present in the query. +- Enhanced the `assess-migration` and `analyze-schema` commands to report issues in Functions or Procedures for variables declared with reference types (%TYPE) in the **Unsupported PL/pgSQL Objects** section. +- Added support to report DDL issues present in the PL/pgSQL blocks of objects listed in the **Unsupported PL/pgSQL Objects** section of the `assess-migration` and `analyze-schema` commands. +- Allow yb-voyager upgrades during migration from the recent breaking release (v1.8.5) to later versions. +- Modified the internal HTTP port to dynamically use an available free port instead of defaulting to 8080, avoiding conflicts with commonly used services. +- Added a guardrail check to the `assess-migration` command to verify that the `pg_stat_statements` extension is properly loaded in the source database. + +### Bug fixes + +- Fixed an [issue](https://github.com/yugabyte/yb-voyager/issues/1895) where NOT VALID constraints in the schema could cause constraint violation errors during data imports; these constraints are now created during the post-snapshot import phase. +- Fixed formatting issues in the assessment HTML report, where extra spaces or characters appeared after the **Unsupported PL/pgSQL Objects** heading, depending on the browser used for viewing. +- Fixed an [issue](https://github.com/yugabyte/yb-voyager/issues/1913) of segmentation faults when certain commands are executed before migration initialization. + +## v1.8.6 - November 26, 2024 + +### New Features + +- Unsupported PL/pgSQL objects detection. Migration assessment and schema analysis commands can now detect and report SQL features and constructs in PL/pgSQL objects in the source schema that are not supported by YugabyteDB. This includes detecting advisory locks, system columns, and XML functions. Voyager reports individual queries in these objects that contain unsupported constructs, such as queries in PL/pgSQL blocks for functions and procedures, or select statements in views and materialized views. + +### Enhancements + +- Using the arguments `--table-list` and `--exclude-table-list` in guardrails now checks for PostgreSQL export to determine which tables require permission checks. +- Added a check for Java as a dependency in guardrails for PostgreSQL export during live migration. +- Added check to verify if [pg_stat_statements](https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/extension-pgstatstatements/) is in a schema not included in the specified `schema_list` and if the migration user has access to queries in the pg_stat_statements view. This is part of the guardrails for assess-migration for PostgreSQL. +- Introduced the `--version` flag in the voyager installer script, which can be used to specify the version to install. +- Added argument [--truncate-tables](https://docs.yugabyte.com/preview/yugabyte-voyager/reference/data-migration/import-data/#arguments) to import data to target for truncating tables, applicable only when --start-clean is true. +- Added support in the assess-migration command to detect the `XMLTABLE()` function under unsupported query constructs. +- Added support for reporting unsupported indexes on some data types, such as daterange, int4range, int8range, tsrange, tstzrange, numrange, and interval, in analyze-schema and assess-migration. +- Added support for reporting unsupported primary and unique key constraints on various data types in assess-migration and analyze-schema. + +### Bug fixes + +- Fixed an [issue](https://github.com/yugabyte/yb-voyager/issues/1920) where export-data errors out if background metadata queries (count*) are still running after pg_dump completes. +- Fixed a bug where the assess-migration command fails when gathering metadata for unsupported query constructs if the pg_stat_statements extension was installed in a non-public schema. +- Fixed nil pointer exceptions and index-out-of-range issues when running export data status and get data-migration-report commands before export data is properly started. +- Fixed a bug in export data status command for accurate status reporting of partition tables during PostgreSQL data export. + +## v1.8.5 - November 12, 2024 + +### Enhancements + +- The guardrail checks to validate source/target database permissions, verify binary dependencies, and check database version compatibility for PostgreSQL in all voyager commands are now enabled by default. +- UI/UX improvements in the PostgreSQL permission grant script (`yb-voyager-pg-grant-migration-permissions.sql`) and new checks are added for replication slots, foreign keys, and triggers in PostgreSQL guardrails. +- Object names are scrollable in the analyze schema HTML report for improved navigation. +- Added constraint names and their corresponding table names when reporting unsupported features related to deferrable and exclusion constraints. +- Added reporting for the REFERENCING clause for triggers and BEFORE ROW triggers on partitioned tables in the analyze-schema and assess-migration reports. +- Added documentation links for unsupported query constructs in the assessment report. +- Standardized the format of data sent to the yugabyted control plane via the assess-migration command, ensuring consistent presentation across various sections of the report, such as Unsupported Features, Unsupported Datatypes, and Unsupported Query Constructs. + +### Bug fixes + +- Fixed the import-schema DDL parsing issue for functions and procedures, where extra spaces before the DDL caused it to be treated as normal DDL, preventing the PLPGSQL parsing logic from triggering. +- Fixed an issue which resulted in "token too long" errors in export-data-from-target when log level was set to DEBUG. + +### Known issues + +- The [assess-migration](https://docs.yugabyte.com/preview/yugabyte-voyager/reference/assess-migration/) command will fail if the [pg_stat_statements](https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/extension-pgstatstatements/) extension is created in a non-public schema, due to the "Unsupported Query Constructs" feature. +To bypass this issue, set the environment variable `REPORT_UNSUPPORTED_QUERY_CONSTRUCTS=false`, which disables the "Unsupported Query Constructs" feature and proceeds with the command execution. + ## v1.8.4 - October 29, 2024 ### New Features From 7990563b192851469a1f26168bf506556f217fd6 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Wed, 11 Dec 2024 21:37:26 +0530 Subject: [PATCH 048/105] Test and Update MInVersionsFixedIn for issue of ALTER PARTITIONED TABLE ADD PRIMARY KEY (#2047) - Added unit test - Updated MinimumVersionsFixedIn --- .../tests/analyze-schema/expected_issues.json | 11 --- .../expectedAssessmentReport.json | 12 +--- yb-voyager/src/query/queryissue/issues_ddl.go | 6 ++ .../src/query/queryissue/issues_ddl_test.go | 70 +++++++++++++++---- yb-voyager/src/ybversion/constants.go | 11 +++ 5 files changed, 75 insertions(+), 35 deletions(-) diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index b836663537..80033ea624 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1319,17 +1319,6 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/3944", "MinimumVersionsFixedIn": null }, - { - "IssueType": "migration_caveats", - "ObjectType": "TABLE", - "ObjectName": "public.range_columns_partition_test", - "Reason": "Adding primary key to a partitioned table is not supported yet.", - "SqlStatement": "ALTER TABLE ONLY public.range_columns_partition_test\n ADD CONSTRAINT range_columns_partition_test_pkey PRIMARY KEY (a, b);", - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", - "Suggestion": "", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/10074", - "MinimumVersionsFixedIn": null - }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 262923e0d7..0b48d75621 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -1960,17 +1960,7 @@ "MigrationCaveats": [ { "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [ - { - "ObjectName": "public.sales_region", - "SqlStatement": "ALTER TABLE ONLY public.sales_region\n ADD CONSTRAINT sales_region_pkey PRIMARY KEY (id, region);" - }, - { - "ObjectName": "schema2.sales_region", - "SqlStatement": "ALTER TABLE ONLY schema2.sales_region\n ADD CONSTRAINT sales_region_pkey PRIMARY KEY (id, region);" - } - ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", + "Objects": [], "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", "MinimumVersionsFixedIn": null }, diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 48bf327e57..390cdf31dd 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) var generatedColumnsIssue = issue.Issue{ @@ -241,6 +242,11 @@ var alterTableAddPKOnPartitionIssue = issue.Issue{ TypeName: "Adding primary key to a partitioned table is not supported yet.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", GH: "https://github.com/yugabyte/yugabyte-db/issues/10074", + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ + ybversion.SERIES_2024_1: ybversion.V2024_1_0_0, + ybversion.SERIES_2024_2: ybversion.V2024_2_0_0, + ybversion.SERIES_2_23: ybversion.V2_23_0_0, + }, } func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index da47f1d141..30aa679633 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -18,28 +18,31 @@ import ( "context" "fmt" "os" + "strings" "testing" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) var ( - yugabytedbContainer *yugabytedb.Container - yugabytedbConnStr string - versions = []string{} + testYugabytedbContainer *yugabytedb.Container + testYugabytedbConnStr string + testYbVersion *ybversion.YBVersion ) func getConn() (*pgx.Conn, error) { ctx := context.Background() var connStr string var err error - if yugabytedbConnStr != "" { - connStr = yugabytedbConnStr + if testYugabytedbConnStr != "" { + connStr = testYugabytedbConnStr } else { - connStr, err = yugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") + connStr, err = testYugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") if err != nil { return nil, err } @@ -53,14 +56,31 @@ func getConn() (*pgx.Conn, error) { return conn, nil } +func fatalIfError(t *testing.T, err error) { + if err != nil { + t.Fatalf("error: %v", err) + } +} + +func assertErrorCorrectlyThrownForIssueForYBVersion(t *testing.T, execErr error, expectedError string, issue issue.Issue) { + isFixed, err := issue.IsFixedIn(testYbVersion) + fatalIfError(t, err) + + if isFixed { + assert.NoError(t, execErr) + } else { + assert.ErrorContains(t, execErr, expectedError) + } +} + func getConnWithNoticeHandler(noticeHandler func(*pgconn.PgConn, *pgconn.Notice)) (*pgx.Conn, error) { ctx := context.Background() var connStr string var err error - if yugabytedbConnStr != "" { - connStr = yugabytedbConnStr + if testYugabytedbConnStr != "" { + connStr = testYugabytedbConnStr } else { - connStr, err = yugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") + connStr, err = testYugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") if err != nil { return nil, err } @@ -128,23 +148,44 @@ func testUnloggedTableIssue(t *testing.T) { } +func testAlterTableAddPKOnPartitionIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE orders2 ( + order_id bigint NOT NULL, + order_date timestamp + ) PARTITION BY RANGE (order_date); + ALTER TABLE orders2 ADD PRIMARY KEY (order_id,order_date)`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "changing primary key of a partitioned table is not yet implemented", alterTableAddPKOnPartitionIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { + var err error ybVersion := os.Getenv("YB_VERSION") if ybVersion == "" { panic("YB_VERSION env variable is not set. Set YB_VERSIONS=2024.1.3.0-b105 for example") } - yugabytedbConnStr = os.Getenv("YB_CONN_STR") - if yugabytedbConnStr == "" { + ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] + testYbVersion, err = ybversion.NewYBVersion(ybVersionWithoutBuild) + fatalIfError(t, err) + + testYugabytedbConnStr = os.Getenv("YB_CONN_STR") + if testYugabytedbConnStr == "" { // spawn yugabytedb container var err error ctx := context.Background() - yugabytedbContainer, err = yugabytedb.Run( + testYugabytedbContainer, err = yugabytedb.Run( ctx, "yugabytedb/yugabyte:"+ybVersion, ) assert.NoError(t, err) - defer yugabytedbContainer.Terminate(context.Background()) + defer testYugabytedbContainer.Terminate(context.Background()) } // run tests @@ -158,4 +199,7 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "unlogged table", ybVersion), testUnloggedTableIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "alter table add PK on partition", ybVersion), testAlterTableAddPKOnPartitionIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/ybversion/constants.go b/yb-voyager/src/ybversion/constants.go index 62650292c1..94bbfffa94 100644 --- a/yb-voyager/src/ybversion/constants.go +++ b/yb-voyager/src/ybversion/constants.go @@ -27,11 +27,18 @@ const ( var LatestStable *YBVersion +var V2024_1_0_0 *YBVersion var V2024_1_3_1 *YBVersion var V2024_2_0_0 *YBVersion +var V2_23_0_0 *YBVersion + func init() { var err error + V2024_1_0_0, err = NewYBVersion("2024.1.0.0") + if err != nil { + panic("could not create version 2024.1.0.0") + } V2024_1_3_1, err = NewYBVersion("2024.1.3.1") if err != nil { panic("could not create version 2024.1.3.1") @@ -41,5 +48,9 @@ func init() { panic("could not create version 2024.2.0.0") } + V2_23_0_0, err = NewYBVersion("2.23.0.0") + if err != nil { + panic("could not create version 2.23.0.0") + } LatestStable = V2024_2_0_0 } From e5e8f9cf2ad2e4d7f1d99a74346690d1ca29dea4 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 12 Dec 2024 11:33:25 +0530 Subject: [PATCH 049/105] Send target-db-version, MinimumVersionsFixedIn details for assessment/analyze-schema reports to controlplane/callhome (#2049) In assess-migration - yugabyted control plane send target-db-version include MinFixVersion for all issues - callhome send target-db-version In analyze-schema - yugabyted control plane (Already done because we send the entire schema report) send target-db-version include MinFixVersion for all issues - callhome send target-db-version --- yb-voyager/cmd/analyzeSchema.go | 22 ++++---- yb-voyager/cmd/assessMigrationCommand.go | 58 ++++++++++++--------- yb-voyager/cmd/common.go | 18 ++++--- yb-voyager/src/callhome/diagnostics.go | 31 ++++++----- yb-voyager/src/callhome/diagnostics_test.go | 31 ++++++----- 5 files changed, 88 insertions(+), 72 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 33c45caac4..1e30245a2d 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -626,15 +626,16 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil } return utils.Issue{ - ObjectType: issueInstance.ObjectType, - ObjectName: issueInstance.ObjectName, - Reason: issueInstance.TypeName, - SqlStatement: issueInstance.SqlStatement, - DocsLink: issueInstance.DocsLink, - FilePath: fileName, - IssueType: issueType, - Suggestion: issueInstance.Suggestion, - GH: issueInstance.GH, + ObjectType: issueInstance.ObjectType, + ObjectName: issueInstance.ObjectName, + Reason: issueInstance.TypeName, + SqlStatement: issueInstance.SqlStatement, + DocsLink: issueInstance.DocsLink, + FilePath: fileName, + IssueType: issueType, + Suggestion: issueInstance.Suggestion, + GH: issueInstance.GH, + MinimumVersionsFixedIn: issueInstance.MinimumVersionsFixedIn, } } @@ -1217,7 +1218,8 @@ func packAndSendAnalyzeSchemaPayload(status string) { } analyzePayload := callhome.AnalyzePhasePayload{ - Issues: callhome.MarshalledJsonString(callhomeIssues), + TargetDBVersion: schemaAnalysisReport.TargetDBVersion, + Issues: callhome.MarshalledJsonString(callhomeIssues), DatabaseObjects: callhome.MarshalledJsonString(lo.Map(schemaAnalysisReport.SchemaSummary.DBObjects, func(dbObject utils.DBObject, _ int) utils.DBObject { dbObject.ObjectNames = "" return dbObject diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index b158ef8f15..90fa21cff9 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -163,6 +163,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { }) assessPayload := callhome.AssessMigrationPhasePayload{ + TargetDBVersion: assessmentReport.TargetDBVersion, MigrationComplexity: assessmentReport.MigrationComplexity, UnsupportedFeatures: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedFeatures, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature { var objects []string @@ -473,6 +474,7 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE payload := AssessMigrationPayload{ PayloadVersion: ASSESS_MIGRATION_PAYLOAD_VERSION, VoyagerVersion: assessmentReport.VoyagerVersion, + TargetDBVersion: assessmentReport.TargetDBVersion, MigrationComplexity: assessmentReport.MigrationComplexity, SchemaSummary: assessmentReport.SchemaSummary, AssessmentIssues: assessmentIssues, @@ -525,13 +527,14 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, unsupportedFeature := range ar.UnsupportedFeatures { for _, object := range unsupportedFeature.Objects { issues = append(issues, AssessmentIssuePayload{ - Type: FEATURE, - TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, - Subtype: unsupportedFeature.FeatureName, - SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: unsupportedFeature.DocsLink, + Type: FEATURE, + TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, + Subtype: unsupportedFeature.FeatureName, + SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: unsupportedFeature.DocsLink, + MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn, }) } } @@ -539,37 +542,40 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, migrationCaveat := range ar.MigrationCaveats { for _, object := range migrationCaveat.Objects { issues = append(issues, AssessmentIssuePayload{ - Type: MIGRATION_CAVEATS, - TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, - Subtype: migrationCaveat.FeatureName, - SubtypeDescription: migrationCaveat.FeatureDescription, - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: migrationCaveat.DocsLink, + Type: MIGRATION_CAVEATS, + TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, + Subtype: migrationCaveat.FeatureName, + SubtypeDescription: migrationCaveat.FeatureDescription, + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: migrationCaveat.DocsLink, + MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn, }) } } for _, uqc := range ar.UnsupportedQueryConstructs { issues = append(issues, AssessmentIssuePayload{ - Type: QUERY_CONSTRUCT, - TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, - Subtype: uqc.ConstructTypeName, - SqlStatement: uqc.Query, - DocsLink: uqc.DocsLink, + Type: QUERY_CONSTRUCT, + TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, + Subtype: uqc.ConstructTypeName, + SqlStatement: uqc.Query, + DocsLink: uqc.DocsLink, + MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn, }) } for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { for _, object := range plpgsqlObjects.Objects { issues = append(issues, AssessmentIssuePayload{ - Type: PLPGSQL_OBJECT, - TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, - Subtype: plpgsqlObjects.FeatureName, - SubtypeDescription: plpgsqlObjects.FeatureDescription, - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: plpgsqlObjects.DocsLink, + Type: PLPGSQL_OBJECT, + TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, + Subtype: plpgsqlObjects.FeatureName, + SubtypeDescription: plpgsqlObjects.FeatureDescription, + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: plpgsqlObjects.DocsLink, + MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn, }) } } diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 3e608da7c7..dd6d459da7 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1224,6 +1224,7 @@ type AssessMigrationDBConfig struct { type AssessMigrationPayload struct { PayloadVersion string VoyagerVersion string + TargetDBVersion *ybversion.YBVersion MigrationComplexity string SchemaSummary utils.SchemaSummary AssessmentIssues []AssessmentIssuePayload @@ -1235,13 +1236,14 @@ type AssessMigrationPayload struct { } type AssessmentIssuePayload struct { - Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC - TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type - Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc - SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype - ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC) - SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC) - DocsLink string `json:"DocsLink"` // docs link based on the subtype + Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC + TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type + Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc + SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype + ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC) + SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC) + DocsLink string `json:"DocsLink"` // docs link based on the subtype + MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"` // key: series (2024.1, 2.21, etc) // Store Type-specific details - extensible, can refer any struct Details json.RawMessage `json:"Details,omitempty"` @@ -1268,7 +1270,7 @@ type TargetSizingRecommendations struct { TotalShardedSize int64 } -var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.0" +var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.1" //====== AssesmentReport struct methods ======// diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 5b65b609f0..8e53134162 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -30,6 +30,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) // call-home json formats @@ -103,18 +104,19 @@ type UnsupportedFeature struct { } type AssessMigrationPhasePayload struct { - MigrationComplexity string `json:"migration_complexity"` - UnsupportedFeatures string `json:"unsupported_features"` - UnsupportedDatatypes string `json:"unsupported_datatypes"` - UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` - MigrationCaveats string `json:"migration_caveats"` - UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` - Error string `json:"error,omitempty"` // Removed it for now, TODO - TableSizingStats string `json:"table_sizing_stats"` - IndexSizingStats string `json:"index_sizing_stats"` - SchemaSummary string `json:"schema_summary"` - SourceConnectivity bool `json:"source_connectivity"` - IopsInterval int64 `json:"iops_interval"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + MigrationComplexity string `json:"migration_complexity"` + UnsupportedFeatures string `json:"unsupported_features"` + UnsupportedDatatypes string `json:"unsupported_datatypes"` + UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` + MigrationCaveats string `json:"migration_caveats"` + UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` + Error string `json:"error,omitempty"` // Removed it for now, TODO + TableSizingStats string `json:"table_sizing_stats"` + IndexSizingStats string `json:"index_sizing_stats"` + SchemaSummary string `json:"schema_summary"` + SourceConnectivity bool `json:"source_connectivity"` + IopsInterval int64 `json:"iops_interval"` } type AssessMigrationBulkPhasePayload struct { @@ -139,8 +141,9 @@ type ExportSchemaPhasePayload struct { // SHOULD NOT REMOVE THESE TWO (issues, database_objects) FIELDS of AnalyzePhasePayload as parsing these specifically here // https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L563 type AnalyzePhasePayload struct { - Issues string `json:"issues"` - DatabaseObjects string `json:"database_objects"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + Issues string `json:"issues"` + DatabaseObjects string `json:"database_objects"` } type ExportDataPhasePayload struct { ParallelJobs int64 `json:"parallel_jobs"` diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index 2575a8e405..a212004a05 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) func TestCallhomeStructs(t *testing.T) { @@ -67,18 +68,19 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate AssessMigrationPhasePayload Struct Definition", actualType: reflect.TypeOf(AssessMigrationPhasePayload{}), expectedType: struct { - MigrationComplexity string `json:"migration_complexity"` - UnsupportedFeatures string `json:"unsupported_features"` - UnsupportedDatatypes string `json:"unsupported_datatypes"` - UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` - MigrationCaveats string `json:"migration_caveats"` - UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` - Error string `json:"error,omitempty"` - TableSizingStats string `json:"table_sizing_stats"` - IndexSizingStats string `json:"index_sizing_stats"` - SchemaSummary string `json:"schema_summary"` - SourceConnectivity bool `json:"source_connectivity"` - IopsInterval int64 `json:"iops_interval"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + MigrationComplexity string `json:"migration_complexity"` + UnsupportedFeatures string `json:"unsupported_features"` + UnsupportedDatatypes string `json:"unsupported_datatypes"` + UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` + MigrationCaveats string `json:"migration_caveats"` + UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` + Error string `json:"error,omitempty"` + TableSizingStats string `json:"table_sizing_stats"` + IndexSizingStats string `json:"index_sizing_stats"` + SchemaSummary string `json:"schema_summary"` + SourceConnectivity bool `json:"source_connectivity"` + IopsInterval int64 `json:"iops_interval"` }{}, }, { @@ -113,8 +115,9 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate AnalyzePhasePayload Struct Definition", actualType: reflect.TypeOf(AnalyzePhasePayload{}), expectedType: struct { - Issues string `json:"issues"` - DatabaseObjects string `json:"database_objects"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + Issues string `json:"issues"` + DatabaseObjects string `json:"database_objects"` }{}, }, { From 5faf23caa9e96d357828b585303bbf360f5d88a7 Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Fri, 13 Dec 2024 10:13:25 +0530 Subject: [PATCH 050/105] Added the PG sample schemas to the GH Actions (#2068) --- .github/workflows/misc-migtests.yml | 5 ++++- .github/workflows/pg-migtests.yml | 27 ++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index c56e4a00d2..93dc69c4d2 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -12,7 +12,7 @@ jobs: services: postgres: - image: postgres:13 + image: postgres:14 env: POSTGRES_PASSWORD: secret # Set health checks to wait until postgres has started @@ -108,6 +108,9 @@ jobs: echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" + echo "TEST: PG sample schemas (omnibus)" + migtests/scripts/run-schema-migration.sh pg/omnibus + echo "Setup the gcp credentials" migtests/scripts/gcs/create_gcs_credentials_file diff --git a/.github/workflows/pg-migtests.yml b/.github/workflows/pg-migtests.yml index cb2457f128..4a1c1d262a 100644 --- a/.github/workflows/pg-migtests.yml +++ b/.github/workflows/pg-migtests.yml @@ -52,9 +52,10 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - - name: "Enable postgres with wal_level as logical" + - name: "Enable postgres with wal_level as logical and install postgis" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" + docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3" docker restart ${{ job.services.postgres.id }} sleep 10 @@ -106,6 +107,30 @@ jobs: echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" + - name: "TEST: PG sample schemas (sakila)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/sakila + + - name: "TEST: PG sample schemas (osm)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/osm + + - name: "TEST: PG sample schemas (adventureworks)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/adventureworks + + - name: "TEST: PG sample schemas (sample-is)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/sample-is + + - name: "TEST: PG sample schemas (pgtbrus)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/pgtbrus\ + + - name: "TEST: PG sample schemas (stackexchange)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/stackexchange + - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/table-list-flags-tests From e563b23c76729585bb3b00bfd9b21a54341b9f8a Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Fri, 13 Dec 2024 10:27:17 +0530 Subject: [PATCH 051/105] Updated the YugabyteDB versions in the tests (#2070) --- .github/workflows/misc-migtests.yml | 2 +- .github/workflows/mysql-migtests.yml | 2 +- .github/workflows/pg-9-migtests.yml | 2 +- .github/workflows/pg-migtests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 93dc69c4d2..cb05a9fbb5 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -86,7 +86,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }} if: ${{ !cancelled() }} run: | - versions=("2.20.5.0-b72" "2.21.1.0-b271" "2024.1.1.0-b137") + versions=("2024.2.0.0-b145" "2.20.8.0-b53" "2024.1.3.1-b8" "2.23.1.0-b220") for version in "${versions[@]}"; do echo "Running tests on version $version" diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index d6b434a57b..81c7ff2ae8 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -10,7 +10,7 @@ jobs: run-mysql-migration-tests: strategy: matrix: - version: [2.21.1.0-b271, 2024.1.1.0-b137, 2.20.5.0-b72] + version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] BETA_FAST_DATA_EXPORT: [0, 1] env: BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index 458804c934..f7b36652fe 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -10,7 +10,7 @@ jobs: run-pg-migration-tests: strategy: matrix: - version: [2.21.1.0-b271] + version: [2024.2.0.0-b145] BETA_FAST_DATA_EXPORT: [0, 1] env: BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} diff --git a/.github/workflows/pg-migtests.yml b/.github/workflows/pg-migtests.yml index 4a1c1d262a..0681dd543e 100644 --- a/.github/workflows/pg-migtests.yml +++ b/.github/workflows/pg-migtests.yml @@ -10,7 +10,7 @@ jobs: run-pg-migration-tests: strategy: matrix: - version: [2.21.1.0-b271, 2024.1.1.0-b137, 2.20.5.0-b72] + version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] BETA_FAST_DATA_EXPORT: [0, 1] test_group: - offline From 75ab89e3b069c21c1bb75c7992d433f9bb12a5b6 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Fri, 13 Dec 2024 10:55:02 +0530 Subject: [PATCH 052/105] ALTER TABLE test cases (#2069) --- .../src/query/queryissue/issues_ddl_test.go | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 30aa679633..bc9beebf27 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -164,6 +164,84 @@ func testAlterTableAddPKOnPartitionIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "changing primary key of a partitioned table is not yet implemented", alterTableAddPKOnPartitionIssue) } +func testSetAttributeIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE public.event_search ( + event_id text, + room_id text, + sender text, + key text, + vector tsvector, + origin_server_ts bigint, + stream_ordering bigint + ); + ALTER TABLE ONLY public.event_search ALTER COLUMN room_id SET (n_distinct=-0.01)`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE ALTER column not supported yet", setAttributeIssue) +} + +func testClusterOnIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE test(ID INT PRIMARY KEY NOT NULL, + Name TEXT NOT NULL, + Age INT NOT NULL, + Address CHAR(50), + Salary REAL); + + CREATE UNIQUE INDEX test_age_salary ON public.test USING btree (age ASC NULLS LAST, salary ASC NULLS LAST); + + ALTER TABLE public.test CLUSTER ON test_age_salary`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE CLUSTER not supported yet", clusterOnIssue) +} + +func testDisableRuleIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + create table trule (a int); + + create rule trule_rule as on update to trule do instead nothing; + + ALTER TABLE trule DISABLE RULE trule_rule`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE DISABLE RULE not supported yet", disableRuleIssue) +} + +func testStorageParameterIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE public.example ( + name text, + email text, + new_id integer NOT NULL, + id2 integer NOT NULL, + CONSTRAINT example_name_check CHECK ((char_length(name) > 3)) + ); + + ALTER TABLE ONLY public.example + ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor = 70);`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "unrecognized parameter", storageParameterIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -202,4 +280,16 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "alter table add PK on partition", ybVersion), testAlterTableAddPKOnPartitionIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "set attribute", ybVersion), testSetAttributeIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "cluster on", ybVersion), testClusterOnIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "disable rule", ybVersion), testDisableRuleIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "storage parameter", ybVersion), testStorageParameterIssue) + assert.True(t, success) + } From 7578488089c0b8c75fe25dcb377f49faf8656570 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:19:23 +0530 Subject: [PATCH 053/105] Added a few more commands to the exportDirInitialisedCheckNeededList to ensure they don't throw a segmentation fault error when run on an empty export dir (#2072) --- yb-voyager/cmd/root.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yb-voyager/cmd/root.go b/yb-voyager/cmd/root.go index 67002b631d..602f1613b8 100644 --- a/yb-voyager/cmd/root.go +++ b/yb-voyager/cmd/root.go @@ -183,7 +183,13 @@ func startPprofServer() { } var exportDirInitialisedCheckNeededList = []string{ + "yb-voyager analyze-schema", + "yb-voyager import data", + "yb-voyager import data to target", + "yb-voyager import data to source", + "yb-voyager import data to source-replica", "yb-voyager import data status", + "yb-voyager export data from target", "yb-voyager export data status", "yb-voyager cutover status", "yb-voyager get data-migration-report", From 5fec48bd8ef4da7d501628fcdc98831ebb37eb89 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 13 Dec 2024 16:33:29 +0530 Subject: [PATCH 054/105] Fixed guardrail check for super user permission and the query for checking if schema exist in case of YB Aeon (#2051) Fixing a few bugs with import schema and YB Aeon type of target db, the difference in behavior is because of yb_superuser with YB Aeon which has a subset of superuser's privileges. Issues are detailed in this ticket https://yugabyte.atlassian.net/browse/DB-14398 --- yb-voyager/cmd/importSchema.go | 2 +- yb-voyager/src/tgtdb/yugabytedb.go | 59 +++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index a3e154688b..388535a2e3 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -479,7 +479,7 @@ func createTargetSchemas(conn *pgx.Conn) { } func checkIfTargetSchemaExists(conn *pgx.Conn, targetSchema string) bool { - checkSchemaExistQuery := fmt.Sprintf("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s'", targetSchema) + checkSchemaExistQuery := fmt.Sprintf("select nspname from pg_namespace n where n.nspname = '%s'", targetSchema) var fetchedSchema string err := conn.QueryRow(context.Background(), checkSchemaExistQuery).Scan(&fetchedSchema) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 58f7f1b70a..baa39102ee 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -1390,24 +1390,55 @@ func IsCurrentUserSuperUser(tconf *TargetConf) (bool, error) { } defer conn.Close(context.Background()) - query := "SELECT rolsuper FROM pg_roles WHERE rolname=current_user" - rows, err := conn.Query(context.Background(), query) - if err != nil { - return false, fmt.Errorf("querying if user is superuser: %w", err) - } - defer rows.Close() - - var isSuperUser bool - if rows.Next() { - err = rows.Scan(&isSuperUser) + runQueryAndCheckPrivilege := func(query string) (bool, error) { + rows, err := conn.Query(context.Background(), query) if err != nil { - return false, fmt.Errorf("scanning row for superuser: %w", err) + return false, fmt.Errorf("querying if user is superuser: %w", err) } - } else { - return false, fmt.Errorf("no current user found in pg_roles") + defer rows.Close() + + var isProperUser bool + if rows.Next() { + err = rows.Scan(&isProperUser) + if err != nil { + return false, fmt.Errorf("scanning row for query: %w", err) + } + } else { + return false, fmt.Errorf("no current user found in pg_roles") + } + return isProperUser, nil + } + + //This rolsuper is set to true in the pg_roles if a user is super user + isSuperUserquery := "SELECT rolsuper FROM pg_roles WHERE rolname=current_user" + + isSuperUser, err := runQueryAndCheckPrivilege(isSuperUserquery) + if err != nil { + return false, fmt.Errorf("error checking super user privilege: %w", err) + } + if isSuperUser { + return true, nil + } + //In case of YugabyteDB Aeon deployment of target database we need to verify if yb_superuser is granted or not + isYbSuperUserQuery := `SELECT + CASE + WHEN EXISTS ( + SELECT 1 + FROM pg_auth_members m + JOIN pg_roles grantee ON m.member = grantee.oid + JOIN pg_roles granted ON m.roleid = granted.oid + WHERE grantee.rolname = CURRENT_USER AND granted.rolname = 'yb_superuser' + ) + THEN TRUE + ELSE FALSE + END AS is_yb_superuser;` + + isYBSuperUser, err := runQueryAndCheckPrivilege(isYbSuperUserQuery) + if err != nil { + return false, fmt.Errorf("error checking yb_superuser privilege: %w", err) } - return isSuperUser, nil + return isYBSuperUser, nil } func (yb *TargetYugabyteDB) GetEnabledTriggersAndFks() (enabledTriggers []string, enabledFks []string, err error) { From 13a678b26f11de103c091084181bd9c64aba688f Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 13 Dec 2024 21:47:34 +0530 Subject: [PATCH 055/105] Reporting the Advisory locks, XML Functions and System columns on the DDLs in the unsupported features in assess/analyze (#2061) - Added three more features to `Unsupported features` in the reports Advisory locks, XML functions and System columns in the DDL. - Moved the MVIEW/VIEW issues from Unsupported PL/pgSQL objects to Unsupported Features. - Minor format change of constraint reporting for PK/Unique on complex datatypes issue - Added unit tests for `GetDDLIssues()` in the `parser_issue_detector_test.go` Fixes https://github.com/yugabyte/yb-voyager/issues/2025 --- .../dummy-export-dir/schema/tables/table.sql | 9 +- .../tests/analyze-schema/expected_issues.json | 62 ++++++--- migtests/tests/analyze-schema/summary.json | 4 +- .../expectedAssessmentReport.json | 99 +++++++------- .../expected_schema_analysis_report.json | 32 ++--- .../expectedAssessmentReport.json | 72 +++++----- .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ .../expectedAssessmentReport.json | 15 +++ yb-voyager/cmd/assessMigrationCommand.go | 6 + .../src/query/queryissue/detectors_ddl.go | 72 +++++----- .../query/queryissue/parser_issue_detector.go | 117 +++++++++------- ..._test.go => parser_issue_detector_test.go} | 125 +++++++++++++++--- .../src/query/queryparser/ddl_processor.go | 32 ++++- .../src/query/queryparser/helpers_struct.go | 35 ----- 20 files changed, 523 insertions(+), 262 deletions(-) rename yb-voyager/src/query/queryissue/{unsupported_ddls_test.go => parser_issue_detector_test.go} (53%) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 3af667c481..5452b47f25 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -375,4 +375,11 @@ CREATE TABLE public.locations ( id integer NOT NULL, name character varying(100), geom geometry(Point,4326) - ); \ No newline at end of file + ); + + CREATE TABLE public.xml_data_example ( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + description XML DEFAULT xmlparse(document 'Default Product100.00Electronics'), + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 80033ea624..2830a4f596 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -20,6 +20,28 @@ "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.xml_data_example", + "Reason": "XML Functions", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);", + "Suggestion": "", + "GH": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "public.xml_data_example", + "Reason": "Unsupported datatype - xml on column - description", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);", + "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "INDEX", @@ -780,7 +802,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl, constraint: combined_tbl_j_key", + "ObjectName": "combined_tbl, constraint: (combined_tbl_j_key)", "Reason": "Primary key and Unique constraint on column 'json' not yet supported", "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "Refer to the docs link for the workaround", @@ -791,7 +813,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl, constraint: pk", + "ObjectName": "combined_tbl, constraint: (pk)", "Reason": "Primary key and Unique constraint on column 'macaddr8' not yet supported", "SqlStatement": "create table combined_tbl (\n\tid int,\n\tc cidr,\n\tci circle,\n\tb box,\n\tj json UNIQUE,\n\tl line,\n\tls lseg,\n\tmaddr macaddr,\n\tmaddr8 macaddr8,\n\tp point,\n\tlsn pg_lsn,\n\tp1 path,\n\tp2 polygon,\n\tid1 txid_snapshot,\n\tbitt bit (13),\n\tbittv bit varying(15),\n\tCONSTRAINT pk PRIMARY KEY (id, maddr8)\n);", "Suggestion": "Refer to the docs link for the workaround", @@ -802,7 +824,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl, constraint: combined_tbl_unique", + "ObjectName": "combined_tbl, constraint: (combined_tbl_unique)", "Reason": "Primary key and Unique constraint on column 'bit' not yet supported", "SqlStatement": "ALTER TABLE combined_tbl\n\t\tADD CONSTRAINT combined_tbl_unique UNIQUE(id, bitt);", "Suggestion": "Refer to the docs link for the workaround", @@ -813,7 +835,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl1, constraint: combined_tbl1_i4_key", + "ObjectName": "combined_tbl1, constraint: (combined_tbl1_i4_key)", "Reason": "Primary key and Unique constraint on column 'int4range' not yet supported", "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", "Suggestion": "Refer to the docs link for the workaround", @@ -824,7 +846,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl1, constraint: combined_tbl1_id_t_n_pkey", + "ObjectName": "combined_tbl1, constraint: (combined_tbl1_id_t_n_pkey)", "Reason": "Primary key and Unique constraint on column 'tsrange' not yet supported", "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", "Suggestion": "Refer to the docs link for the workaround", @@ -835,7 +857,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl1, constraint: combined_tbl1_id_t_n_pkey", + "ObjectName": "combined_tbl1, constraint: (combined_tbl1_id_t_n_pkey)", "Reason": "Primary key and Unique constraint on column 'numrange' not yet supported", "SqlStatement": "CREATE TABLE combined_tbl1(\n\tid int,\n\tt tsrange,\n\td daterange,\n\ttz tstzrange,\n\tn numrange,\n\ti4 int4range UNIQUE,\n\ti8 int8range,\n\tinym INTERVAL YEAR TO MONTH,\n\tinds INTERVAL DAY TO SECOND(9),\n\tPRIMARY KEY(id, t, n)\n);", "Suggestion": "Refer to the docs link for the workaround", @@ -846,7 +868,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "combined_tbl1, constraint: combined_tbl1_unique", + "ObjectName": "combined_tbl1, constraint: (combined_tbl1_unique)", "Reason": "Primary key and Unique constraint on column 'daterange' not yet supported", "SqlStatement": "ALTER TABLE combined_tbl1\n\t\tADD CONSTRAINT combined_tbl1_unique UNIQUE(id, d);", "Suggestion": "Refer to the docs link for the workaround", @@ -879,7 +901,7 @@ { "IssueType": "unsupported_features", "ObjectType": "TABLE", - "ObjectName": "test_interval, constraint: test_interval_frequency_pkey", + "ObjectName": "test_interval, constraint: (test_interval_frequency_pkey)", "Reason": "Primary key and Unique constraint on column 'interval' not yet supported", "SqlStatement": "create table test_interval(\n frequency interval primary key,\n\tcol1 int\n);", "Suggestion": "Refer to the docs link for the workaround", @@ -1671,66 +1693,66 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "MVIEW", "ObjectName": "public.sample_data_view", "Reason": "XML Functions", - "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", + "SqlStatement": "CREATE MATERIALIZED VIEW public.sample_data_view AS\n SELECT sample_data.id,\n sample_data.name,\n sample_data.description,\n XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data,\n pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired,\n sample_data.ctid AS row_ctid,\n sample_data.xmin AS xmin_value\n FROM public.sample_data\n WITH NO DATA;", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "MVIEW", "ObjectName": "public.sample_data_view", "Reason": "Advisory Locks", - "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", + "SqlStatement": "CREATE MATERIALIZED VIEW public.sample_data_view AS\n SELECT sample_data.id,\n sample_data.name,\n sample_data.description,\n XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data,\n pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired,\n sample_data.ctid AS row_ctid,\n sample_data.xmin AS xmin_value\n FROM public.sample_data\n WITH NO DATA;", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "MVIEW", "ObjectName": "public.sample_data_view", "Reason": "System Columns", - "SqlStatement": "SELECT sample_data.id, sample_data.name, sample_data.description, xmlforest(sample_data.name AS name, sample_data.description AS description) AS xml_data, pg_try_advisory_lock(sample_data.id::bigint) AS lock_acquired, sample_data.ctid AS row_ctid, sample_data.xmin AS xmin_value FROM public.sample_data", + "SqlStatement": "CREATE MATERIALIZED VIEW public.sample_data_view AS\n SELECT sample_data.id,\n sample_data.name,\n sample_data.description,\n XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data,\n pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired,\n sample_data.ctid AS row_ctid,\n sample_data.xmin AS xmin_value\n FROM public.sample_data\n WITH NO DATA;", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "public.orders_view", "Reason": "XML Functions", - "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", + "SqlStatement": "CREATE VIEW public.orders_view AS\n SELECT orders.order_id,\n orders.customer_name,\n orders.product_name,\n orders.quantity,\n orders.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name), XMLELEMENT(NAME \"Quantity\", orders.quantity), XMLELEMENT(NAME \"TotalPrice\", (orders.price * (orders.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,\n orders.ctid AS row_ctid,\n orders.xmin AS transaction_id\n FROM public.orders;", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "public.orders_view", "Reason": "Advisory Locks", - "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", + "SqlStatement": "CREATE VIEW public.orders_view AS\n SELECT orders.order_id,\n orders.customer_name,\n orders.product_name,\n orders.quantity,\n orders.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name), XMLELEMENT(NAME \"Quantity\", orders.quantity), XMLELEMENT(NAME \"TotalPrice\", (orders.price * (orders.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,\n orders.ctid AS row_ctid,\n orders.xmin AS transaction_id\n FROM public.orders;", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "public.orders_view", "Reason": "System Columns", - "SqlStatement": "SELECT orders.order_id, orders.customer_name, orders.product_name, orders.quantity, orders.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name), xmlelement(name \"Quantity\", orders.quantity), xmlelement(name \"TotalPrice\", orders.price * orders.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", orders.customer_name), xmlelement(name \"Product\", orders.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(orders.customer_name || orders.product_name)::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id FROM public.orders", + "SqlStatement": "CREATE VIEW public.orders_view AS\n SELECT orders.order_id,\n orders.customer_name,\n orders.product_name,\n orders.quantity,\n orders.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name), XMLELEMENT(NAME \"Quantity\", orders.quantity), XMLELEMENT(NAME \"TotalPrice\", (orders.price * (orders.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,\n orders.ctid AS row_ctid,\n orders.xmin AS transaction_id\n FROM public.orders;", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 09b4ec3f28..e047b08fd4 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -24,9 +24,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 49, + "TotalCount": 50, "InvalidCount": 33, - "ObjectNames": "combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, + "ObjectNames": "public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, { "ObjectType": "INDEX", "TotalCount": 43, diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 842f993a55..ceecd2595a 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -648,6 +648,55 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [ + { + "ObjectName": "humanresources.vjobcandidate", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidate.jobcandidateid,\n jobcandidate.businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n jobcandidate.modifieddate\n FROM humanresources.jobcandidate;" + }, + { + "ObjectName": "humanresources.vjobcandidateeducation", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jc.jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;" + }, + { + "ObjectName": "humanresources.vjobcandidateemployment", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidate.jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;" + }, + { + "ObjectName": "person.vadditionalcontactinfo", + "SqlStatement": "CREATE VIEW person.vadditionalcontactinfo AS\n SELECT p.businessentityid,\n p.firstname,\n p.middlename,\n p.lastname,\n (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber,\n btrim((((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1])::character varying)::text) AS telephonespecialinstructions,\n (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street,\n (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city,\n (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince,\n (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode,\n (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion,\n (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions,\n (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress,\n btrim((((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1])::character varying)::text) AS emailspecialinstructions,\n (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber,\n p.rowguid,\n p.modifieddate\n FROM (person.person p\n LEFT JOIN ( SELECT person.businessentityid,\n unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node\n FROM person.person\n WHERE (person.additionalcontactinfo IS NOT NULL)) additional ON ((p.businessentityid = additional.businessentityid)));" + }, + { + "ObjectName": "production.vproductmodelcatalogdescription", + "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodel.productmodelid,\n productmodel.name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n productmodel.rowguid,\n productmodel.modifieddate\n FROM production.productmodel\n WHERE (productmodel.catalogdescription IS NOT NULL);" + }, + { + "ObjectName": "production.vproductmodelinstructions", + "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT pm.productmodelid,\n pm.name,\n ((xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, pm.step))[1])::character varying(1024) AS \"Step\",\n pm.rowguid,\n pm.modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;" + }, + { + "ObjectName": "sales.vpersondemographics", + "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT person.businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (person.demographics IS NOT NULL);" + }, + { + "ObjectName": "sales.vstorewithdemographics", + "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT store.businessentityid,\n store.name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Storage parameters in DDLs", "Objects": [], @@ -1736,53 +1785,5 @@ } ], "UnsupportedQueryConstructs": null, - "UnsupportedPlPgSqlObjects": [ - { - "FeatureName": "XML Functions", - "Objects": [ - { - "ObjectType": "VIEW", - "ObjectName": "humanresources.vjobcandidate", - "SqlStatement": "SELECT jobcandidate.jobcandidateid, jobcandidate.businessentityid, (xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Prefix\", (xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.First\", (xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Middle\", (xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Last\", (xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Suffix\", (xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"Skills\", (xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Addr.Type\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.CountryRegion\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.State\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.City\", (xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(20) AS \"Addr.PostalCode\", (xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"EMail\", (xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"WebSite\", jobcandidate.modifieddate FROM humanresources.jobcandidate" - }, - { - "ObjectType": "VIEW", - "ObjectName": "humanresources.vjobcandidateeducation", - "SqlStatement": "SELECT jc.jobcandidateid, (xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Level\", (xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.StartDate\", (xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.EndDate\", (xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Degree\", (xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Major\", (xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Minor\", (xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPA\", (xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPAScale\", (xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.School\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.CountryRegion\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.State\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.City\" FROM (SELECT unnesting.jobcandidateid, CAST(('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || unnesting.education::varchar::text) || '\u003c/root\u003e'::text AS xml) AS doc FROM (SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education FROM humanresources.jobcandidate) unnesting) jc" - }, - { - "ObjectType": "VIEW", - "ObjectName": "humanresources.vjobcandidateemployment", - "SqlStatement": "SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.StartDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.EndDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.OrgName\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.JobTitle\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Responsibility\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.FunctionCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.IndustryCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.CountryRegion\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.State\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.City\" FROM humanresources.jobcandidate" - }, - { - "ObjectType": "VIEW", - "ObjectName": "person.vadditionalcontactinfo", - "SqlStatement": "SELECT p.businessentityid, p.firstname, p.middlename, p.lastname, (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber, btrim((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS telephonespecialinstructions, (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street, (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city, (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince, (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode, (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion, (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions, (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress, btrim((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS emailspecialinstructions, (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber, p.rowguid, p.modifieddate FROM person.person p LEFT JOIN (SELECT person.businessentityid, unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node FROM person.person WHERE person.additionalcontactinfo IS NOT NULL) additional ON p.businessentityid = additional.businessentityid" - }, - { - "ObjectType": "VIEW", - "ObjectName": "production.vproductmodelcatalogdescription", - "SqlStatement": "SELECT productmodel.productmodelid, productmodel.name, (xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1]::varchar AS \"Summary\", (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar AS manufacturer, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(30) AS copyright, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS producturl, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantyperiod, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantydescription, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS noofyears, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS maintenancedescription, (xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS wheel, (xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS saddle, (xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS pedal, (xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar AS bikeframe, (xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS crankset, (xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS pictureangle, (xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS picturesize, (xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productphotoid, (xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS material, (xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS color, (xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productline, (xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS style, (xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(1024) AS riderexperience, productmodel.rowguid, productmodel.modifieddate FROM production.productmodel WHERE productmodel.catalogdescription IS NOT NULL" - }, - { - "ObjectType": "VIEW", - "ObjectName": "production.vproductmodelinstructions", - "SqlStatement": "SELECT pm.productmodelid, pm.name, (xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1]::varchar AS instructions, (xpath('@LocationID'::text, pm.mfginstructions))[1]::varchar::int AS \"LocationID\", (xpath('@SetupHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"SetupHours\", (xpath('@MachineHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"MachineHours\", (xpath('@LaborHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"LaborHours\", (xpath('@LotSize'::text, pm.mfginstructions))[1]::varchar::int AS \"LotSize\", (xpath('/step/text()'::text, pm.step))[1]::varchar(1024) AS \"Step\", pm.rowguid, pm.modifieddate FROM (SELECT locations.productmodelid, locations.name, locations.rowguid, locations.modifieddate, locations.instructions, locations.mfginstructions, unnest(xpath('step'::text, locations.mfginstructions)) AS step FROM (SELECT productmodel.productmodelid, productmodel.name, productmodel.rowguid, productmodel.modifieddate, productmodel.instructions, unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions FROM production.productmodel) locations) pm" - }, - { - "ObjectType": "VIEW", - "ObjectName": "sales.vpersondemographics", - "SqlStatement": "SELECT person.businessentityid, (xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::money AS totalpurchaseytd, (xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS datefirstpurchase, (xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS birthdate, (xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS maritalstatus, (xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS yearlyincome, (xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS gender, (xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS totalchildren, (xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numberchildrenathome, (xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS education, (xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS occupation, (xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::boolean AS homeownerflag, (xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numbercarsowned FROM person.person WHERE person.demographics IS NOT NULL" - }, - { - "ObjectType": "VIEW", - "ObjectName": "sales.vstorewithdemographics", - "SqlStatement": "SELECT store.businessentityid, store.name, unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualSales\", unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualRevenue\", unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"BankName\", unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(5) AS \"BusinessType\", unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"YearOpened\", unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"Specialty\", unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"SquareFeet\", unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Brands\", unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Internet\", unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"NumberEmployees\" FROM sales.store" - } - ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", - "MinimumVersionsFixedIn": null - } - ] + "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index 5e8900e503..728396eb1c 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -972,11 +972,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidate", "Reason": "XML Functions", - "SqlStatement": "SELECT jobcandidate.jobcandidateid, jobcandidate.businessentityid, (xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Prefix\", (xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.First\", (xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Middle\", (xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Last\", (xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Name.Suffix\", (xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"Skills\", (xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(30) AS \"Addr.Type\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.CountryRegion\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.State\", (xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(100) AS \"Addr.Loc.City\", (xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar(20) AS \"Addr.PostalCode\", (xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"EMail\", (xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1]::varchar AS \"WebSite\", jobcandidate.modifieddate FROM humanresources.jobcandidate", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidate.jobcandidateid,\n jobcandidate.businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n jobcandidate.modifieddate\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -984,11 +984,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidateeducation", "Reason": "XML Functions", - "SqlStatement": "SELECT jc.jobcandidateid, (xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Level\", (xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.StartDate\", (xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(20)::date AS \"Edu.EndDate\", (xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Degree\", (xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Major\", (xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(50) AS \"Edu.Minor\", (xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPA\", (xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(5) AS \"Edu.GPAScale\", (xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.School\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.CountryRegion\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.State\", (xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1]::varchar(100) AS \"Edu.Loc.City\" FROM (SELECT unnesting.jobcandidateid, CAST(('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || unnesting.education::varchar::text) || '\u003c/root\u003e'::text AS xml) AS doc FROM (SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education FROM humanresources.jobcandidate) unnesting) jc", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jc.jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -996,11 +996,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidateemployment", "Reason": "XML Functions", - "SqlStatement": "SELECT jobcandidate.jobcandidateid, unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.StartDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(20)::date AS \"Emp.EndDate\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.OrgName\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar(100) AS \"Emp.JobTitle\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Responsibility\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.FunctionCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.IndustryCategory\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.CountryRegion\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.State\", unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))::varchar AS \"Emp.Loc.City\" FROM humanresources.jobcandidate", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidate.jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1008,11 +1008,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "person.vadditionalcontactinfo", "Reason": "XML Functions", - "SqlStatement": "SELECT p.businessentityid, p.firstname, p.middlename, p.lastname, (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber, btrim((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS telephonespecialinstructions, (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street, (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city, (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince, (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode, (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion, (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions, (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress, btrim((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1]::varchar::text) AS emailspecialinstructions, (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber, p.rowguid, p.modifieddate FROM person.person p LEFT JOIN (SELECT person.businessentityid, unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node FROM person.person WHERE person.additionalcontactinfo IS NOT NULL) additional ON p.businessentityid = additional.businessentityid", + "SqlStatement": "CREATE VIEW person.vadditionalcontactinfo AS\n SELECT p.businessentityid,\n p.firstname,\n p.middlename,\n p.lastname,\n (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber,\n btrim((((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1])::character varying)::text) AS telephonespecialinstructions,\n (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street,\n (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city,\n (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince,\n (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode,\n (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion,\n (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions,\n (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress,\n btrim((((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1])::character varying)::text) AS emailspecialinstructions,\n (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber,\n p.rowguid,\n p.modifieddate\n FROM (person.person p\n LEFT JOIN ( SELECT person.businessentityid,\n unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node\n FROM person.person\n WHERE (person.additionalcontactinfo IS NOT NULL)) additional ON ((p.businessentityid = additional.businessentityid)));", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1020,11 +1020,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "production.vproductmodelcatalogdescription", "Reason": "XML Functions", - "SqlStatement": "SELECT productmodel.productmodelid, productmodel.name, (xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1]::varchar AS \"Summary\", (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar AS manufacturer, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(30) AS copyright, (xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS producturl, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantyperiod, (xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS warrantydescription, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS noofyears, (xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1]::varchar(256) AS maintenancedescription, (xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS wheel, (xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS saddle, (xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS pedal, (xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar AS bikeframe, (xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1]::varchar(256) AS crankset, (xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS pictureangle, (xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS picturesize, (xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productphotoid, (xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS material, (xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS color, (xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS productline, (xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(256) AS style, (xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1]::varchar(1024) AS riderexperience, productmodel.rowguid, productmodel.modifieddate FROM production.productmodel WHERE productmodel.catalogdescription IS NOT NULL", + "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodel.productmodelid,\n productmodel.name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n productmodel.rowguid,\n productmodel.modifieddate\n FROM production.productmodel\n WHERE (productmodel.catalogdescription IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1032,11 +1032,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "production.vproductmodelinstructions", "Reason": "XML Functions", - "SqlStatement": "SELECT pm.productmodelid, pm.name, (xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1]::varchar AS instructions, (xpath('@LocationID'::text, pm.mfginstructions))[1]::varchar::int AS \"LocationID\", (xpath('@SetupHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"SetupHours\", (xpath('@MachineHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"MachineHours\", (xpath('@LaborHours'::text, pm.mfginstructions))[1]::varchar::numeric(9, 4) AS \"LaborHours\", (xpath('@LotSize'::text, pm.mfginstructions))[1]::varchar::int AS \"LotSize\", (xpath('/step/text()'::text, pm.step))[1]::varchar(1024) AS \"Step\", pm.rowguid, pm.modifieddate FROM (SELECT locations.productmodelid, locations.name, locations.rowguid, locations.modifieddate, locations.instructions, locations.mfginstructions, unnest(xpath('step'::text, locations.mfginstructions)) AS step FROM (SELECT productmodel.productmodelid, productmodel.name, productmodel.rowguid, productmodel.modifieddate, productmodel.instructions, unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions FROM production.productmodel) locations) pm", + "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT pm.productmodelid,\n pm.name,\n ((xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, pm.step))[1])::character varying(1024) AS \"Step\",\n pm.rowguid,\n pm.modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1044,11 +1044,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "sales.vpersondemographics", "Reason": "XML Functions", - "SqlStatement": "SELECT person.businessentityid, (xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::money AS totalpurchaseytd, (xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS datefirstpurchase, (xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::date AS birthdate, (xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS maritalstatus, (xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS yearlyincome, (xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(1) AS gender, (xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS totalchildren, (xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numberchildrenathome, (xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS education, (xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar(30) AS occupation, (xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::boolean AS homeownerflag, (xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1]::varchar::int AS numbercarsowned FROM person.person WHERE person.demographics IS NOT NULL", + "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT person.businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (person.demographics IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1056,11 +1056,11 @@ "MinimumVersionsFixedIn": null }, { - "IssueType": "unsupported_plpgsql_objects", + "IssueType": "unsupported_features", "ObjectType": "VIEW", "ObjectName": "sales.vstorewithdemographics", "Reason": "XML Functions", - "SqlStatement": "SELECT store.businessentityid, store.name, unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualSales\", unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::money AS \"AnnualRevenue\", unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"BankName\", unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(5) AS \"BusinessType\", unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"YearOpened\", unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(50) AS \"Specialty\", unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"SquareFeet\", unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Brands\", unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar(30) AS \"Internet\", unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[]))::varchar::int AS \"NumberEmployees\" FROM sales.store", + "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT store.businessentityid,\n store.name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 0b48d75621..7bac4c30e1 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -522,15 +522,15 @@ "FeatureName": "Primary / Unique key constraints on complex datatypes", "Objects": [ { - "ObjectName": "public.combined_tbl, constraint: combined_tbl_bittv_key", + "ObjectName": "public.combined_tbl, constraint: (combined_tbl_bittv_key)", "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT combined_tbl_bittv_key UNIQUE (bittv);" }, { - "ObjectName": "public.combined_tbl, constraint: uk", + "ObjectName": "public.combined_tbl, constraint: (uk)", "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT uk UNIQUE (lsn);" }, { - "ObjectName": "public.combined_tbl, constraint: combined_tbl_pkey", + "ObjectName": "public.combined_tbl, constraint: (combined_tbl_pkey)", "SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n ADD CONSTRAINT combined_tbl_pkey PRIMARY KEY (id, arr_enum);" } ], @@ -551,7 +551,40 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", "MinimumVersionsFixedIn": null - } + }, + { + "FeatureName": "System Columns", + "Objects": [ + { + "ObjectName": "public.ordersentry_view", + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "XML Functions", + "Objects": [ + { + "ObjectName": "public.ordersentry_view", + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Advisory Locks", + "Objects": [ + { + "ObjectName": "public.ordersentry_view", + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -2189,39 +2222,10 @@ "ObjectType": "FUNCTION", "ObjectName": "schema2.process_order", "SqlStatement": "SELECT pg_advisory_unlock(orderid);" - }, - { - "ObjectType": "VIEW", - "ObjectName": "public.ordersentry_view", - "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "XML Functions", - "Objects": [ - { - "ObjectType": "VIEW", - "ObjectName": "public.ordersentry_view", - "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" - } - ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "System Columns", - "Objects": [ - { - "ObjectType": "VIEW", - "ObjectName": "public.ordersentry_view", - "SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry" - } - ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", - "MinimumVersionsFixedIn": null - } + } ] } \ No newline at end of file diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 8e10a3d80d..77a3e8bdfe 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -248,6 +248,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [], diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 6c6a0be1e4..f14e1b0e5a 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -493,6 +493,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [ diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 5fdd452359..b3d67ae3c7 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -73,6 +73,21 @@ ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [ diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index 8c511ac3cc..a8222d3a4b 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -80,6 +80,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [], diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 560cd9403b..d7fd60bcc6 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -377,6 +377,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [], diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index 184099368b..0c474ddcf9 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -118,6 +118,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [ diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index cb0333a968..59c7edd001 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -76,6 +76,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [], diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index f6be2c83ed..682f4b0983 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -82,6 +82,21 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ + { + "FeatureName": "Advisory Locks", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName":"System Columns", + "Objects":[], + "MinimumVersionsFixedIn":null + }, + { + "FeatureName": "XML Functions", + "Objects": [], +"MinimumVersionsFixedIn":null + }, { "FeatureName": "GIST indexes", "Objects": [], diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 90fa21cff9..30d6cd1b14 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -951,6 +951,9 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea var minVersionsFixedInSet bool for _, issue := range schemaAnalysisReport.Issues { + if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, issue.IssueType) { + continue + } if strings.Contains(issue.Reason, issueReason) { objectInfo := ObjectInfo{ ObjectName: issue.ObjectName, @@ -1000,6 +1003,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("Advisory Locks", "Advisory Locks", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("XML Functions", "XML Functions", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("System Columns", "System Columns", schemaAnalysisReport, false, "")) return unsupportedFeatures, nil } diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index f2b6b2f9ff..33714d00c6 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -50,7 +50,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss // Check for generated columns if len(table.GeneratedColumns) > 0 { issues = append(issues, NewGeneratedColumnsIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", // query string table.GeneratedColumns, @@ -60,7 +60,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss // Check for unlogged table if table.IsUnlogged { issues = append(issues, NewUnloggedTableIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", // query string )) @@ -68,7 +68,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if table.IsInherited { issues = append(issues, NewInheritanceIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", )) @@ -79,7 +79,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss for _, c := range table.Constraints { if c.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { issues = append(issues, NewExclusionConstraintIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), "", )) @@ -87,7 +87,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if c.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && c.IsDeferrable { issues = append(issues, NewDeferrableConstraintIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), "", )) @@ -105,8 +105,8 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss continue } issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( - TABLE_OBJECT_TYPE, - fmt.Sprintf("%s, constraint: %s", table.GetObjectName(), c.ConstraintName), + obj.GetObjectType(), + fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), "", typeName, true, @@ -129,10 +129,10 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss isUnsupportedDatatypeInLiveWithFFOrFB := isUnsupportedDatatypeInLiveWithFFOrFBList || isUDTDatatype || isArrayOfEnumsDatatype if isUnsupportedDatatype { - reportUnsupportedDatatypes(col, TABLE_OBJECT_TYPE, table.GetObjectName(), &issues) + reportUnsupportedDatatypes(col, obj.GetObjectType(), table.GetObjectName(), &issues) } else if isUnsupportedDatatypeInLive { issues = append(issues, NewUnsupportedDatatypesForLMIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", col.TypeName, @@ -145,7 +145,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss reportTypeName = fmt.Sprintf("%s[]", reportTypeName) } issues = append(issues, NewUnsupportedDatatypesForLMWithFFOrFBIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", reportTypeName, @@ -165,7 +165,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss alterAddPk := d.primaryConsInAlter[table.GetObjectName()] if alterAddPk != nil { issues = append(issues, NewAlterTableAddPKOnPartiionIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), alterAddPk.Query, )) @@ -175,7 +175,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if table.IsExpressionPartition && (len(primaryKeyColumns) > 0 || len(uniqueKeyColumns) > 0) { issues = append(issues, NewExpressionPartitionIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", )) @@ -184,7 +184,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if table.PartitionStrategy == queryparser.LIST_PARTITION && len(table.PartitionColumns) > 1 { issues = append(issues, NewMultiColumnListPartition( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", )) @@ -192,7 +192,7 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss partitionColumnsNotInPK, _ := lo.Difference(table.PartitionColumns, primaryKeyColumns) if len(primaryKeyColumns) > 0 && len(partitionColumnsNotInPK) > 0 { issues = append(issues, NewInsufficientColumnInPKForPartition( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), table.GetObjectName(), "", partitionColumnsNotInPK, @@ -252,7 +252,7 @@ func (f *ForeignTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q issues := make([]QueryIssue, 0) issues = append(issues, NewForeignTableIssue( - FOREIGN_TABLE_OBJECT_TYPE, + obj.GetObjectType(), foreignTable.GetObjectName(), "", foreignTable.ServerName, @@ -261,7 +261,7 @@ func (f *ForeignTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q for _, col := range foreignTable.Columns { isUnsupportedDatatype := utils.ContainsAnyStringFromSlice(srcdb.PostgresUnsupportedDataTypes, col.TypeName) if isUnsupportedDatatype { - reportUnsupportedDatatypes(col, FOREIGN_TABLE_OBJECT_TYPE, foreignTable.GetObjectName(), &issues) + reportUnsupportedDatatypes(col, obj.GetObjectType(), foreignTable.GetObjectName(), &issues) } } @@ -287,7 +287,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss // Check for unsupported index methods if slices.Contains(UnsupportedIndexMethods, index.AccessMethod) { issues = append(issues, NewUnsupportedIndexMethodIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", // query string index.AccessMethod, @@ -297,7 +297,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss // Check for storage parameters if index.NumStorageOptions > 0 { issues = append(issues, NewStorageParameterIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", // query string )) @@ -307,7 +307,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if index.AccessMethod == GIN_ACCESS_METHOD { if len(index.Params) > 1 { issues = append(issues, NewMultiColumnGinIndexIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", )) @@ -316,7 +316,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss param := index.Params[0] if param.SortByOrder != queryparser.DEFAULT_SORTING_ORDER { issues = append(issues, NewOrderedGinIndexIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", )) @@ -341,7 +341,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss isUDTType := slices.Contains(d.compositeTypes, param.GetFullExprCastTypeName()) if param.IsExprCastArrayType { issues = append(issues, NewIndexOnComplexDatatypesIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", "array", @@ -352,7 +352,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss reportTypeName = "user_defined_type" } issues = append(issues, NewIndexOnComplexDatatypesIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", reportTypeName, @@ -366,7 +366,7 @@ func (d *IndexIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss continue } issues = append(issues, NewIndexOnComplexDatatypesIssue( - INDEX_OBJECT_TYPE, + obj.GetObjectType(), index.GetObjectName(), "", typeName, @@ -397,7 +397,7 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q case queryparser.SET_OPTIONS: if alter.NumSetAttributes > 0 { issues = append(issues, NewSetAttributeIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), alter.GetObjectName(), "", // query string )) @@ -405,21 +405,21 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q case queryparser.ADD_CONSTRAINT: if alter.NumStorageOptions > 0 { issues = append(issues, NewStorageParameterIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), alter.GetObjectName(), "", // query string )) } if alter.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { issues = append(issues, NewExclusionConstraintIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), "", )) } if alter.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && alter.IsDeferrable { issues = append(issues, NewDeferrableConstraintIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), "", )) @@ -428,7 +428,7 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q if alter.ConstraintType == queryparser.PRIMARY_CONSTR_TYPE && aid.partitionTablesMap[alter.GetObjectName()] { issues = append(issues, NewAlterTableAddPKOnPartiionIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), alter.GetObjectName(), "", )) @@ -446,8 +446,8 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q continue } issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( - TABLE_OBJECT_TYPE, - fmt.Sprintf("%s, constraint: %s", alter.GetObjectName(), alter.ConstraintName), + obj.GetObjectType(), + fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), "", typeName, false, @@ -457,14 +457,14 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q } case queryparser.DISABLE_RULE: issues = append(issues, NewDisableRuleIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), alter.GetObjectName(), "", // query string alter.RuleName, )) case queryparser.CLUSTER_ON: issues = append(issues, NewClusterONIssue( - TABLE_OBJECT_TYPE, + obj.GetObjectType(), alter.GetObjectName(), "", // query string )) @@ -486,7 +486,7 @@ func (p *PolicyIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIs issues := make([]QueryIssue, 0) if len(policy.RoleNames) > 0 { issues = append(issues, NewPolicyRoleIssue( - POLICY_OBJECT_TYPE, + obj.GetObjectType(), policy.GetObjectName(), "", policy.RoleNames, @@ -511,7 +511,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer if trigger.IsConstraint { issues = append(issues, NewConstraintTriggerIssue( - TRIGGER_OBJECT_TYPE, + obj.GetObjectType(), trigger.GetObjectName(), "", )) @@ -519,7 +519,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer if trigger.NumTransitionRelations > 0 { issues = append(issues, NewReferencingClauseTrigIssue( - TRIGGER_OBJECT_TYPE, + obj.GetObjectType(), trigger.GetObjectName(), "", )) @@ -527,7 +527,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer if trigger.IsBeforeRowTrigger() && tid.partitionTablesMap[trigger.GetTableName()] { issues = append(issues, NewBeforeRowOnPartitionTableIssue( - TRIGGER_OBJECT_TYPE, + obj.GetObjectType(), trigger.GetObjectName(), "", )) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 27d28fa936..2fc872014b 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -93,10 +93,9 @@ func (p *ParserIssueDetector) getAllIssues(query string) ([]QueryIssue, error) { if err != nil { return nil, fmt.Errorf("error getting plpgsql issues: %v", err) } - dmlIssues, err := p.getDMLIssues(query) if err != nil { - return nil, fmt.Errorf("error getting dml issues: %v", err) + return nil, fmt.Errorf("error getting generic issues: %v", err) } ddlIssues, err := p.getDDLIssues(query) if err != nil { @@ -133,59 +132,40 @@ func (p *ParserIssueDetector) getPLPGSQLIssues(query string) ([]QueryIssue, erro if err != nil { return nil, fmt.Errorf("error parsing query: %w", err) } - //TODO handle this in DDLPARSER, DDLIssueDetector - if queryparser.IsPLPGSQLObject(parseTree) { - objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) - plpgsqlQueries, err := queryparser.GetAllPLPGSQLStatements(query) - if err != nil { - return nil, fmt.Errorf("error getting all the queries from query: %w", err) - } - var issues []QueryIssue - for _, plpgsqlQuery := range plpgsqlQueries { - issuesInQuery, err := p.getAllIssues(plpgsqlQuery) - if err != nil { - //there can be plpgsql expr queries no parseable via parser e.g. "withdrawal > balance" - log.Errorf("error getting issues in query-%s: %v", query, err) - continue - } - issues = append(issues, issuesInQuery...) - } - percentTypeSyntaxIssues, err := p.GetPercentTypeSyntaxIssues(query) - if err != nil { - return nil, fmt.Errorf("error getting reference TYPE syntax issues: %v", err) - } - issues = append(issues, percentTypeSyntaxIssues...) - - return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { - //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object - //e.g. replacing the DML_QUERY and "" to FUNCTION and - i.ObjectType = objType - i.ObjectName = objName - return i - }), nil - } - //Handle the Mview/View DDL's Select stmt issues - if queryparser.IsViewObject(parseTree) || queryparser.IsMviewObject(parseTree) { - objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) - selectStmtQuery, err := queryparser.GetSelectStmtQueryFromViewOrMView(parseTree) - if err != nil { - return nil, fmt.Errorf("error deparsing a select stmt: %v", err) - } - issues, err := p.getDMLIssues(selectStmtQuery) + if !queryparser.IsPLPGSQLObject(parseTree) { + return nil, nil + } + //TODO handle this in DDLPARSER, DDLIssueDetector + objType, objName := queryparser.GetObjectTypeAndObjectName(parseTree) + plpgsqlQueries, err := queryparser.GetAllPLPGSQLStatements(query) + if err != nil { + return nil, fmt.Errorf("error getting all the queries from query: %w", err) + } + var issues []QueryIssue + for _, plpgsqlQuery := range plpgsqlQueries { + issuesInQuery, err := p.getAllIssues(plpgsqlQuery) if err != nil { - return nil, err + //there can be plpgsql expr queries no parseable via parser e.g. "withdrawal > balance" + log.Errorf("error getting issues in query-%s: %v", query, err) + continue } + issues = append(issues, issuesInQuery...) + } - return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { - //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object - //e.g. replacing the DML_QUERY and "" to FUNCTION and - i.ObjectType = objType - i.ObjectName = objName - return i - }), nil + percentTypeSyntaxIssues, err := p.GetPercentTypeSyntaxIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting reference TYPE syntax issues: %v", err) } - return nil, nil + issues = append(issues, percentTypeSyntaxIssues...) + + return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { + //Replacing the objectType and objectName to the original ObjectType and ObjectName of the PLPGSQL object + //e.g. replacing the DML_QUERY and "" to FUNCTION and + i.ObjectType = objType + i.ObjectName = objName + return i + }), nil } func (p *ParserIssueDetector) ParseRequiredDDLs(query string) error { @@ -266,6 +246,13 @@ func (p *ParserIssueDetector) getDDLIssues(query string) ([]QueryIssue, error) { if err != nil { return nil, fmt.Errorf("error parsing a query: %v", err) } + isDDL, err := queryparser.IsDDL(parseTree) + if err != nil { + return nil, fmt.Errorf("error checking if query is ddl: %w", err) + } + if !isDDL { + return nil, nil + } // Parse the query into a DDL object ddlObj, err := queryparser.ProcessDDL(parseTree) if err != nil { @@ -289,6 +276,23 @@ func (p *ParserIssueDetector) getDDLIssues(query string) ([]QueryIssue, error) { issues[i].SqlStatement = query } } + + /* + For detecting these generic issues (Advisory locks, XML functions and System columns as of now) on DDL example - + CREATE INDEX idx_invoices on invoices (xpath('/invoice/customer/text()', data)); + We need to call it on DDLs as well + */ + genericIssues, err := p.genericIssues(query) + if err != nil { + return nil, fmt.Errorf("error getting generic issues: %w", err) + } + + for _, i := range genericIssues { + //In case of genericIssues we don't populate the proper obj type and obj name + i.ObjectType = ddlObj.GetObjectType() + i.ObjectName = ddlObj.GetObjectName() + issues = append(issues, i) + } return issues, nil } @@ -331,7 +335,6 @@ func (p *ParserIssueDetector) GetDMLIssues(query string, targetDbVersion *ybvers return p.getIssuesNotFixedInTargetDbVersion(issues, targetDbVersion) } -// TODO: in future when we will DDL issues detection here we need `GetDDLIssues` func (p *ParserIssueDetector) getDMLIssues(query string) ([]QueryIssue, error) { parseTree, err := queryparser.Parse(query) if err != nil { @@ -345,6 +348,18 @@ func (p *ParserIssueDetector) getDMLIssues(query string) ([]QueryIssue, error) { //Skip all the DDLs coming to this function return nil, nil } + issues, err := p.genericIssues(query) + if err != nil { + return issues, err + } + return issues, err +} + +func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) { + parseTree, err := queryparser.Parse(query) + if err != nil { + return nil, fmt.Errorf("error parsing query: %w", err) + } var result []QueryIssue var unsupportedConstructs []string visited := make(map[protoreflect.Message]bool) diff --git a/yb-voyager/src/query/queryissue/unsupported_ddls_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go similarity index 53% rename from yb-voyager/src/query/queryissue/unsupported_ddls_test.go rename to yb-voyager/src/query/queryissue/parser_issue_detector_test.go index a04f209846..e9330a09e4 100644 --- a/yb-voyager/src/query/queryissue/unsupported_ddls_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -19,6 +19,8 @@ import ( "slices" "testing" + "github.com/google/go-cmp/cmp" + "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" @@ -118,9 +120,50 @@ $$ LANGUAGE plpgsql;` );` stmt12 = `CREATE TABLE test_dt (id int, d daterange);` stmt13 = `CREATE INDEX idx_on_daterange on test_dt (d);` + stmt14 = `CREATE MATERIALIZED VIEW public.sample_data_view AS + SELECT sample_data.id, + sample_data.name, + sample_data.description, + XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data, + pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired, + sample_data.ctid AS row_ctid, + sample_data.xmin AS xmin_value + FROM public.sample_data + WITH NO DATA;` + stmt15 = `CREATE VIEW public.orders_view AS + SELECT orders.order_id, + orders.customer_name, + orders.product_name, + orders.quantity, + orders.price, + XMLELEMENT(NAME "OrderDetails", XMLELEMENT(NAME "Customer", orders.customer_name), XMLELEMENT(NAME "Product", orders.product_name), XMLELEMENT(NAME "Quantity", orders.quantity), XMLELEMENT(NAME "TotalPrice", (orders.price * (orders.quantity)::numeric))) AS order_xml, + XMLCONCAT(XMLELEMENT(NAME "Customer", orders.customer_name), XMLELEMENT(NAME "Product", orders.product_name)) AS summary_xml, + pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired, + orders.ctid AS row_ctid, + orders.xmin AS transaction_id + FROM public.orders + WITH LOCAL CHECK OPTION;` + stmt16 = `CREATE TABLE public.xml_data_example ( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + d daterange Unique, + description XML DEFAULT xmlparse(document 'Default Product100.00Electronics'), + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +) PARTITION BY LIST(id, name);` + stmt17 = `ALTER TABLE invoices +ADD CONSTRAINT valid_invoice_structure +CHECK (xpath_exists('/invoice/customer', data));` + stmt18 = `CREATE INDEX idx_invoices on invoices (xpath('/invoice/customer/text()', data));` ) -func TestAllDDLIssues(t *testing.T) { +func modifyiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { + return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { + i.ObjectType = objType + i.ObjectName = objName + return i + }) +} +func TestAllIssues(t *testing.T) { requiredDDLs := []string{stmt12} parserIssueDetector := NewParserIssueDetector() stmtsWithExpectedIssues := map[string][]QueryIssue{ @@ -128,19 +171,19 @@ func TestAllDDLIssues(t *testing.T) { NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "public.emp1.salary%TYPE"), NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.name%TYPE"), NewPercentTypeSyntaxIssue("FUNCTION", "list_high_earners", "employees.salary%TYPE"), - NewClusterONIssue("FUNCTION", "list_high_earners", "ALTER TABLE employees CLUSTER ON idx;"), - NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(sender_id);"), - NewAdvisoryLocksIssue("FUNCTION", "list_high_earners", "SELECT pg_advisory_unlock(receiver_id);"), - NewXmlFunctionsIssue("FUNCTION", "list_high_earners", "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;"), - NewSystemColumnsIssue("FUNCTION", "list_high_earners", "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);"), + NewClusterONIssue("TABLE", "employees", "ALTER TABLE employees CLUSTER ON idx;"), + NewAdvisoryLocksIssue("DML_QUERY", "", "SELECT pg_advisory_unlock(sender_id);"), + NewAdvisoryLocksIssue("DML_QUERY", "", "SELECT pg_advisory_unlock(receiver_id);"), + NewXmlFunctionsIssue("DML_QUERY", "", "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;"), + NewSystemColumnsIssue("DML_QUERY", "", "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);"), }, stmt2: []QueryIssue{ NewPercentTypeSyntaxIssue("FUNCTION", "process_order", "orders.id%TYPE"), - NewStorageParameterIssue("FUNCTION", "process_order", "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70);"), - NewUnloggedTableIssue("FUNCTION", "process_order", "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);"), - NewMultiColumnGinIndexIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON example_table USING gin(name, name1);"), - NewUnsupportedIndexMethodIssue("FUNCTION", "process_order", "CREATE INDEX idx_example ON schema1.example_table USING gist(name);", "gist"), - NewAdvisoryLocksIssue("FUNCTION", "process_order", "SELECT pg_advisory_unlock(orderid);"), + NewStorageParameterIssue("TABLE", "public.example", "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70);"), + NewUnloggedTableIssue("TABLE", "tbl_unlog", "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);"), + NewMultiColumnGinIndexIssue("INDEX", "idx_example ON example_table", "CREATE INDEX idx_example ON example_table USING gin(name, name1);"), + NewUnsupportedIndexMethodIssue("INDEX", "idx_example ON schema1.example_table", "CREATE INDEX idx_example ON schema1.example_table USING gist(name);", "gist"), + NewAdvisoryLocksIssue("DML_QUERY", "", "SELECT pg_advisory_unlock(orderid);"), }, stmt3: []QueryIssue{ NewStorageParameterIssue("INDEX", "abc ON public.example", stmt3), @@ -176,6 +219,11 @@ func TestAllDDLIssues(t *testing.T) { }, } + //Should modify it in similar way we do it actual code as the particular DDL issue in plpgsql can have different Details map on the basis of objectType + stmtsWithExpectedIssues[stmt1] = modifyiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt1], "FUNCTION", "list_high_earners") + + stmtsWithExpectedIssues[stmt2] = modifyiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt2], "FUNCTION", "process_order") + for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) assert.NoError(t, err, "Error parsing required ddl: %s", stmt) @@ -186,15 +234,58 @@ func TestAllDDLIssues(t *testing.T) { assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) for _, expectedIssue := range expectedIssues { - found := slices.ContainsFunc(issues, func(QueryIssue QueryIssue) bool { - typeNameMatches := QueryIssue.TypeName == expectedIssue.TypeName - queryMatches := QueryIssue.SqlStatement == expectedIssue.SqlStatement - objectNameMatches := QueryIssue.ObjectName == expectedIssue.ObjectName - objectTypeMatches := QueryIssue.ObjectType == expectedIssue.ObjectType - return typeNameMatches && queryMatches && objectNameMatches && objectTypeMatches + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) }) assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) } } } + +func TestDDLIssues(t *testing.T) { + requiredDDLs := []string{stmt16} + parserIssueDetector := NewParserIssueDetector() + stmtsWithExpectedIssues := map[string][]QueryIssue{ + stmt14: []QueryIssue{ + NewAdvisoryLocksIssue("MVIEW", "public.sample_data_view", stmt14), + NewSystemColumnsIssue("MVIEW", "public.sample_data_view", stmt14), + NewXmlFunctionsIssue("MVIEW", "public.sample_data_view", stmt14), + }, + stmt15: []QueryIssue{ + NewAdvisoryLocksIssue("VIEW", "public.orders_view", stmt15), + NewSystemColumnsIssue("VIEW", "public.orders_view", stmt15), + NewXmlFunctionsIssue("VIEW", "public.orders_view", stmt15), + //TODO: Add CHECK OPTION issue when we move it from regex to parser logic + }, + stmt16: []QueryIssue{ + NewXmlFunctionsIssue("TABLE", "public.xml_data_example", stmt16), + NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue("TABLE", "public.xml_data_example, constraint: (xml_data_example_d_key)", stmt16, "daterange", true), + NewMultiColumnListPartition("TABLE", "public.xml_data_example", stmt16), + NewInsufficientColumnInPKForPartition("TABLE", "public.xml_data_example", stmt16, []string{"name"}), + NewXMLDatatypeIssue("TABLE", "public.xml_data_example", stmt16, "description"), + }, + stmt17: []QueryIssue{ + NewXmlFunctionsIssue("TABLE", "invoices", stmt17), + }, + stmt18: []QueryIssue{ + NewXmlFunctionsIssue("INDEX", "idx_invoices ON invoices", stmt18), + }, + } + for _, stmt := range requiredDDLs { + err := parserIssueDetector.ParseRequiredDDLs(stmt) + assert.NoError(t, err, "Error parsing required ddl: %s", stmt) + } + for stmt, expectedIssues := range stmtsWithExpectedIssues { + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index f7249d8896..43b82db893 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -41,6 +41,7 @@ with the required for storing the information which should have these required f */ type DDLObject interface { GetObjectName() string + GetObjectType() string GetSchemaName() string } @@ -322,6 +323,8 @@ func (t *Table) GetObjectName() string { } func (t *Table) GetSchemaName() string { return t.SchemaName } +func (t *Table) GetObjectType() string { return TABLE_OBJECT_TYPE } + func (t *Table) PrimaryKeyColumns() []string { for _, c := range t.Constraints { if c.ConstraintType == PRIMARY_CONSTR_TYPE { @@ -347,7 +350,7 @@ func (t *Table) addConstraint(conType pg_query.ConstrType, columns []string, spe Columns: columns, IsDeferrable: deferrable, } - generatedConName := tc.generateConstraintName(t.GetObjectName()) + generatedConName := tc.generateConstraintName(t.TableName) conName := lo.Ternary(specifiedConName == "", generatedConName, specifiedConName) tc.ConstraintName = conName t.Constraints = append(t.Constraints, tc) @@ -404,6 +407,8 @@ func (f *ForeignTable) GetObjectName() string { } func (f *ForeignTable) GetSchemaName() string { return f.SchemaName } +func (t *ForeignTable) GetObjectType() string { return FOREIGN_TABLE_OBJECT_TYPE } + //===========INDEX PROCESSOR ================================ // IndexProcessor handles parsing CREATE INDEX statements @@ -501,6 +506,8 @@ func (i *Index) GetTableName() string { return lo.Ternary(i.SchemaName != "", fmt.Sprintf("%s.%s", i.SchemaName, i.TableName), i.TableName) } +func (i *Index) GetObjectType() string { return INDEX_OBJECT_TYPE } + //===========ALTER TABLE PROCESSOR ================================ // AlterTableProcessor handles parsing ALTER TABLE statements @@ -609,6 +616,8 @@ func (a *AlterTable) GetObjectName() string { } func (a *AlterTable) GetSchemaName() string { return a.SchemaName } +func (a *AlterTable) GetObjectType() string { return TABLE_OBJECT_TYPE } + func (a *AlterTable) AddPrimaryKeyOrUniqueCons() bool { return a.ConstraintType == PRIMARY_CONSTR_TYPE || a.ConstraintType == UNIQUE_CONSTR_TYPE } @@ -673,6 +682,8 @@ func (p *Policy) GetObjectName() string { } func (p *Policy) GetSchemaName() string { return p.SchemaName } +func (p *Policy) GetObjectType() string { return POLICY_OBJECT_TYPE } + //=====================TRIGGER PROCESSOR ================== // TriggerProcessor handles parsing CREATE Trigger statements @@ -745,6 +756,8 @@ func (t *Trigger) GetTableName() string { func (t *Trigger) GetSchemaName() string { return t.SchemaName } +func (t *Trigger) GetObjectType() string { return TRIGGER_OBJECT_TYPE } + /* e.g.CREATE TRIGGER after_insert_or_delete_trigger @@ -819,6 +832,8 @@ func (c *CreateType) GetObjectName() string { } func (c *CreateType) GetSchemaName() string { return c.SchemaName } +func (c *CreateType) GetObjectType() string { return TYPE_OBJECT_TYPE } + //===========================VIEW PROCESSOR=================== type ViewProcessor struct{} @@ -849,6 +864,8 @@ func (v *View) GetObjectName() string { } func (v *View) GetSchemaName() string { return v.SchemaName } +func (v *View) GetObjectType() string { return VIEW_OBJECT_TYPE } + //===========================MVIEW PROCESSOR=================== type MViewProcessor struct{} @@ -879,6 +896,8 @@ func (mv *MView) GetObjectName() string { } func (mv *MView) GetSchemaName() string { return mv.SchemaName } +func (mv *MView) GetObjectType() string { return MVIEW_OBJECT_TYPE } + //=============================No-Op PROCESSOR ================== //No op Processor for objects we don't have Processor yet @@ -896,6 +915,7 @@ type Object struct { func (o *Object) GetObjectName() string { return o.ObjectName } func (o *Object) GetSchemaName() string { return o.SchemaName } +func (o *Object) GetObjectType() string { return "OBJECT" } func (n *NoOpProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { return &Object{}, nil @@ -932,6 +952,16 @@ func GetDDLProcessor(parseTree *pg_query.ParseResult) (DDLProcessor, error) { } const ( + TABLE_OBJECT_TYPE = "TABLE" + TYPE_OBJECT_TYPE = "TYPE" + VIEW_OBJECT_TYPE = "VIEW" + MVIEW_OBJECT_TYPE = "MVIEW" + FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" + FUNCTION_OBJECT_TYPE = "FUNCTION" + PROCEDURE_OBJECT_TYPE = "PROCEDURE" + INDEX_OBJECT_TYPE = "INDEX" + POLICY_OBJECT_TYPE = "POLICY" + TRIGGER_OBJECT_TYPE = "TRIGGER" ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 45d1db3901..492796392b 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -23,25 +23,6 @@ import ( "github.com/samber/lo" ) -func DeparseSelectStmt(selectStmt *pg_query.SelectStmt) (string, error) { - if selectStmt != nil { - parseResult := &pg_query.ParseResult{ - Stmts: []*pg_query.RawStmt{ - { - Stmt: &pg_query.Node{ - Node: &pg_query.Node_SelectStmt{SelectStmt: selectStmt}, - }, - }, - }, - } - - // Deparse the SelectStmt to get the string representation - selectSQL, err := pg_query.Deparse(parseResult) - return selectSQL, err - } - return "", nil -} - func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) @@ -58,22 +39,6 @@ func IsMviewObject(parseTree *pg_query.ParseResult) bool { return isCreateAsStmt && createAsNode.CreateTableAsStmt.Objtype == pg_query.ObjectType_OBJECT_MATVIEW } -func GetSelectStmtQueryFromViewOrMView(parseTree *pg_query.ParseResult) (string, error) { - viewNode, isViewStmt := getCreateViewNode(parseTree) - createAsNode, _ := getCreateTableAsStmtNode(parseTree) //For MVIEW case - var selectStmt *pg_query.SelectStmt - if isViewStmt { - selectStmt = viewNode.ViewStmt.GetQuery().GetSelectStmt() - } else { - selectStmt = createAsNode.CreateTableAsStmt.GetQuery().GetSelectStmt() - } - selectStmtQuery, err := DeparseSelectStmt(selectStmt) - if err != nil { - return "", fmt.Errorf("deparsing the select stmt: %v", err) - } - return selectStmtQuery, nil -} - func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { createFuncNode, isCreateFunc := getCreateFuncStmtNode(parseTree) viewNode, isViewStmt := getCreateViewNode(parseTree) From dd614282a4df620e70f409a2afdcdf3358191217 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 16 Dec 2024 11:21:32 +0530 Subject: [PATCH 056/105] Update handling of unlogged tables as per 2024.2 (#2074) * Specify MinVersionFixedIn = 2024.2.0.0 * If issue is detected, but filtered out because of target-db-version, add a note in assessment report. * Propagate NOTICE client messages to stdout in import-schema (with the exception of the ALTER TABLE "table rewrite will cause inconsistencies" notice) * Moved some functions from importData.go to importSchemaYugabyteDB.go * Upgrade pgx from v4 to v5 in importSchemaYUgabyteDB.go --- .../tests/analyze-schema/expected_issues.json | 11 - .../expectedAssessmentReport.json | 16 +- yb-voyager/cmd/assessMigrationCommand.go | 8 +- yb-voyager/cmd/importData.go | 118 ----------- yb-voyager/cmd/importSchemaYugabyteDB.go | 200 +++++++++++++++++- yb-voyager/src/query/queryissue/issues_ddl.go | 3 + .../src/query/queryissue/issues_ddl_test.go | 43 +--- .../query/queryissue/parser_issue_detector.go | 8 + .../queryissue/parser_issue_detector_test.go | 17 +- yb-voyager/src/utils/utils.go | 7 +- 10 files changed, 237 insertions(+), 194 deletions(-) diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 2830a4f596..6b44e2beb8 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -534,17 +534,6 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/10652", "MinimumVersionsFixedIn": null }, - { - "IssueType": "unsupported_features", - "ObjectType": "TABLE", - "ObjectName": "tbl_unlogged", - "Reason": "UNLOGGED tables are not supported yet.", - "SqlStatement": "CREATE UNLOGGED TABLE tbl_unlogged (id int, val text);", - "Suggestion": "Remove UNLOGGED keyword to make it work", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/1129/", - "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", - "MinimumVersionsFixedIn": null - }, { "IssueType": "unsupported_features", "ObjectType": "VIEW", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 7bac4c30e1..6021709791 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -539,17 +539,7 @@ }, { "FeatureName": "Unlogged tables", - "Objects": [ - { - "ObjectName": "public.tbl_unlogged", - "SqlStatement": "CREATE UNLOGGED TABLE public.tbl_unlogged (\n id integer,\n val text\n);" - }, - { - "ObjectName": "schema2.tbl_unlogged", - "SqlStatement": "CREATE UNLOGGED TABLE schema2.tbl_unlogged (\n id integer,\n val text\n);" - } - ], - "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", + "Objects": [], "MinimumVersionsFixedIn": null }, { @@ -1989,7 +1979,9 @@ "SizeInBytes": 8192 } ], - "Notes": null, + "Notes": [ + "There are some Unlogged tables in the schema. They will be created as regular LOGGED tables in YugabyteDB as unlogged tables are not supported." + ], "MigrationCaveats": [ { "FeatureName": "Alter partitioned tables to add Primary Key", diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 30d6cd1b14..95270d6f45 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1336,7 +1336,8 @@ To manually modify the schema, please refer: https://github.com/yugabyte/yugabyte-db/issues/7850 so take a look and modify them if not supported.` + GIN_INDEXES = `There are some BITMAP indexes present in the schema that will get converted to GIN indexes, but GIN indexes are partially supported in YugabyteDB as mentioned in https://github.com/yugabyte/yugabyte-db/issues/7850 so take a look and modify them if not supported.` + UNLOGGED_TABLE_NOTE = `There are some Unlogged tables in the schema. They will be created as regular LOGGED tables in YugabyteDB as unlogged tables are not supported.` ) const FOREIGN_TABLE_NOTE = `There are some Foreign tables in the schema, but during the export schema phase, exported schema does not include the SERVER and USER MAPPING objects. Therefore, you must manually create these objects before import schema. For more information on each of them, run analyze-schema. ` @@ -1363,7 +1364,12 @@ func addNotesToAssessmentReport() { } } } + case POSTGRESQL: + if parserIssueDetector.IsUnloggedTablesIssueFiltered { + assessmentReport.Notes = append(assessmentReport.Notes, UNLOGGED_TABLE_NOTE) + } } + } func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration []utils.TableColumnsDataTypes, unsupportedDataTypesForLiveMigrationWithFForFB []utils.TableColumnsDataTypes) { diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 000c4d9519..3331df51b0 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -16,7 +16,6 @@ limitations under the License. package cmd import ( - "context" "fmt" "io" "os" @@ -29,7 +28,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/fatih/color" - "github.com/jackc/pgx/v4" "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/sourcegraph/conc/pool" @@ -1216,65 +1214,6 @@ func importBatch(batch *Batch, importBatchArgsProto *tgtdb.ImportBatchArgs) { } } -func newTargetConn() *pgx.Conn { - conn, err := pgx.Connect(context.Background(), tconf.GetConnectionUri()) - if err != nil { - utils.WaitChannel <- 1 - <-utils.WaitChannel - utils.ErrExit("connect to target db: %s", err) - } - - setTargetSchema(conn) - - if sourceDBType == ORACLE && enableOrafce { - setOrafceSearchPath(conn) - } - - return conn -} - -// TODO: Eventually get rid of this function in favour of TargetYugabyteDB.setTargetSchema(). -func setTargetSchema(conn *pgx.Conn) { - if sourceDBType == POSTGRESQL || tconf.Schema == YUGABYTEDB_DEFAULT_SCHEMA { - // For PG, schema name is already included in the object name. - // No need to set schema if importing in the default schema. - return - } - checkSchemaExistsQuery := fmt.Sprintf("SELECT count(schema_name) FROM information_schema.schemata WHERE schema_name = '%s'", tconf.Schema) - var cntSchemaName int - - if err := conn.QueryRow(context.Background(), checkSchemaExistsQuery).Scan(&cntSchemaName); err != nil { - utils.ErrExit("run query %q on target %q to check schema exists: %s", checkSchemaExistsQuery, tconf.Host, err) - } else if cntSchemaName == 0 { - utils.ErrExit("schema '%s' does not exist in target", tconf.Schema) - } - - setSchemaQuery := fmt.Sprintf("SET SCHEMA '%s'", tconf.Schema) - _, err := conn.Exec(context.Background(), setSchemaQuery) - if err != nil { - utils.ErrExit("run query %q on target %q: %s", setSchemaQuery, tconf.Host, err) - } -} - -func dropIdx(conn *pgx.Conn, idxName string) error { - dropIdxQuery := fmt.Sprintf("DROP INDEX IF EXISTS %s", idxName) - log.Infof("Dropping index: %q", dropIdxQuery) - _, err := conn.Exec(context.Background(), dropIdxQuery) - if err != nil { - return fmt.Errorf("failed to drop index %q: %w", idxName, err) - } - return nil -} - -func setOrafceSearchPath(conn *pgx.Conn) { - // append oracle schema in the search_path for orafce - updateSearchPath := `SELECT set_config('search_path', current_setting('search_path') || ', oracle', false)` - _, err := conn.Exec(context.Background(), updateSearchPath) - if err != nil { - utils.ErrExit("unable to update search_path for orafce extension: %v", err) - } -} - func getIndexName(sqlQuery string, indexName string) (string, error) { // Return the index name itself if it is aleady qualified with schema name if len(strings.Split(indexName, ".")) == 2 { @@ -1292,63 +1231,6 @@ func getIndexName(sqlQuery string, indexName string) (string, error) { return "", fmt.Errorf("could not find `ON` keyword in the CREATE INDEX statement") } -// TODO: need automation tests for this, covering cases like schema(public vs non-public) or case sensitive names -func beforeIndexCreation(sqlInfo sqlInfo, conn **pgx.Conn, objType string) error { - if !strings.Contains(strings.ToUpper(sqlInfo.stmt), "CREATE INDEX") { - return nil - } - - fullyQualifiedObjName, err := getIndexName(sqlInfo.stmt, sqlInfo.objName) - if err != nil { - return fmt.Errorf("extract qualified index name from DDL [%v]: %w", sqlInfo.stmt, err) - } - if invalidTargetIndexesCache == nil { - invalidTargetIndexesCache, err = getInvalidIndexes(conn) - if err != nil { - return fmt.Errorf("failed to fetch invalid indexes: %w", err) - } - } - - // check index valid or not - if invalidTargetIndexesCache[fullyQualifiedObjName] { - log.Infof("index %q already exists but in invalid state, dropping it", fullyQualifiedObjName) - err = dropIdx(*conn, fullyQualifiedObjName) - if err != nil { - return fmt.Errorf("drop invalid index %q: %w", fullyQualifiedObjName, err) - } - } - - // print the index name as index creation takes time and user can see the progress - color.Yellow("creating index %s ...", fullyQualifiedObjName) - return nil -} - -func getInvalidIndexes(conn **pgx.Conn) (map[string]bool, error) { - var result = make(map[string]bool) - // NOTE: this shouldn't fetch any predefined indexes of pg_catalog schema (assuming they can't be invalid) or indexes of other successful migrations - query := "SELECT indexrelid::regclass FROM pg_index WHERE indisvalid = false" - - rows, err := (*conn).Query(context.Background(), query) - if err != nil { - return nil, fmt.Errorf("querying invalid indexes: %w", err) - } - defer rows.Close() - - for rows.Next() { - var fullyQualifiedIndexName string - err := rows.Scan(&fullyQualifiedIndexName) - if err != nil { - return nil, fmt.Errorf("scanning row for invalid index name: %w", err) - } - // if schema is not provided by catalog table, then it is public schema - if !strings.Contains(fullyQualifiedIndexName, ".") { - fullyQualifiedIndexName = fmt.Sprintf("public.%s", fullyQualifiedIndexName) - } - result[fullyQualifiedIndexName] = true - } - return result, nil -} - // TODO: This function is a duplicate of the one in tgtdb/yb.go. Consolidate the two. func getTargetSchemaName(tableName string) string { parts := strings.Split(tableName, ".") diff --git a/yb-voyager/cmd/importSchemaYugabyteDB.go b/yb-voyager/cmd/importSchemaYugabyteDB.go index ecdf28446b..9cb37b96d9 100644 --- a/yb-voyager/cmd/importSchemaYugabyteDB.go +++ b/yb-voyager/cmd/importSchemaYugabyteDB.go @@ -23,7 +23,9 @@ import ( "time" "github.com/fatih/color" - "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/samber/lo" log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" @@ -34,6 +36,10 @@ import ( var deferredSqlStmts []sqlInfo var finalFailedSqlStmts []string +// The client message (NOTICE/WARNING) from psql is stored in this global variable. +// as part of the noticeHandler function for every query executed. +var notice *pgconn.Notice + func importSchemaInternal(exportDir string, importObjectList []string, skipFn func(string, string) bool) error { schemaDir := filepath.Join(exportDir, "schema") @@ -93,14 +99,13 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo } if objType == "TABLE" { - stmt := strings.ToUpper(sqlInfo.stmt) // Check if the statement should be skipped - skip, err := shouldSkipDDL(stmt) + skip, err := shouldSkipDDL(sqlInfo.stmt) if err != nil { - return fmt.Errorf("error checking whether to skip DDL: %v", err) + return fmt.Errorf("error checking whether to skip DDL for statement [%s]: %v", sqlInfo.stmt, err) } if skip { - log.Infof("Skipping DDL: %s", stmt) + log.Infof("Skipping DDL: %s", sqlInfo.stmt) continue } } @@ -114,6 +119,13 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo } func shouldSkipDDL(stmt string) (bool, error) { + stmt = strings.ToUpper(stmt) + + // pg_dump generate `SET client_min_messages = 'warning';`, but we want to get + // NOTICE severity as well (which is the default), hence skipping this. + if strings.Contains(stmt, "SET CLIENT_MIN_MESSAGES") { + return true, nil + } skipReplicaIdentity := strings.Contains(stmt, "ALTER TABLE") && strings.Contains(stmt, "REPLICA IDENTITY") if skipReplicaIdentity { return true, nil @@ -133,6 +145,7 @@ func shouldSkipDDL(stmt string) (bool, error) { func executeSqlStmtWithRetries(conn **pgx.Conn, sqlInfo sqlInfo, objType string) error { var err error + var stmtNotice *pgconn.Notice log.Infof("On %s run query:\n%s\n", tconf.Host, sqlInfo.formattedStmt) for retryCount := 0; retryCount <= DDL_MAX_RETRY_COUNT; retryCount++ { if retryCount > 0 { // Not the first iteration. @@ -149,9 +162,10 @@ func executeSqlStmtWithRetries(conn **pgx.Conn, sqlInfo sqlInfo, objType string) return fmt.Errorf("before index creation: %w", err) } } - _, err = (*conn).Exec(context.Background(), sqlInfo.formattedStmt) + stmtNotice, err = execStmtAndGetNotice(*conn, sqlInfo.formattedStmt) if err == nil { - utils.PrintSqlStmtIfDDL(sqlInfo.stmt, utils.GetObjectFileName(filepath.Join(exportDir, "schema"), objType)) + utils.PrintSqlStmtIfDDL(sqlInfo.stmt, utils.GetObjectFileName(filepath.Join(exportDir, "schema"), objType), + getNoticeMessage(stmtNotice)) return nil } @@ -203,7 +217,8 @@ func executeSqlStmtWithRetries(conn **pgx.Conn, sqlInfo sqlInfo, objType string) if missingRequiredSchemaObject(err) { // Do nothing for deferred case } else { - utils.PrintSqlStmtIfDDL(sqlInfo.stmt, utils.GetObjectFileName(filepath.Join(exportDir, "schema"), objType)) + utils.PrintSqlStmtIfDDL(sqlInfo.stmt, utils.GetObjectFileName(filepath.Join(exportDir, "schema"), objType), + getNoticeMessage(stmtNotice)) color.Red(fmt.Sprintf("%s\n", err.Error())) if tconf.ContinueOnError { log.Infof("appending stmt to failedSqlStmts list: %s\n", utils.GetSqlStmtToPrint(sqlInfo.stmt)) @@ -242,9 +257,14 @@ func importDeferredStatements() { beforeDeferredSqlCount := len(deferredSqlStmts) var failedSqlStmtInIthIteration []string for j := 0; j < len(deferredSqlStmts); j++ { - _, err = conn.Exec(context.Background(), deferredSqlStmts[j].formattedStmt) + var stmtNotice *pgconn.Notice + stmtNotice, err = execStmtAndGetNotice(conn, deferredSqlStmts[j].formattedStmt) if err == nil { utils.PrintAndLog("%s\n", utils.GetSqlStmtToPrint(deferredSqlStmts[j].stmt)) + noticeMsg := getNoticeMessage(stmtNotice) + if noticeMsg != "" { + utils.PrintAndLog(color.YellowString("%s\n", noticeMsg)) + } // removing successfully executed SQL deferredSqlStmts = append(deferredSqlStmts[:j], deferredSqlStmts[j+1:]...) break @@ -311,3 +331,165 @@ func applySchemaObjectFilterFlags(importObjectOrderList []string) []string { } return finalImportObjectList } + +func getInvalidIndexes(conn **pgx.Conn) (map[string]bool, error) { + var result = make(map[string]bool) + // NOTE: this shouldn't fetch any predefined indexes of pg_catalog schema (assuming they can't be invalid) or indexes of other successful migrations + query := "SELECT indexrelid::regclass FROM pg_index WHERE indisvalid = false" + + rows, err := (*conn).Query(context.Background(), query) + if err != nil { + return nil, fmt.Errorf("querying invalid indexes: %w", err) + } + defer rows.Close() + + for rows.Next() { + var fullyQualifiedIndexName string + err := rows.Scan(&fullyQualifiedIndexName) + if err != nil { + return nil, fmt.Errorf("scanning row for invalid index name: %w", err) + } + // if schema is not provided by catalog table, then it is public schema + if !strings.Contains(fullyQualifiedIndexName, ".") { + fullyQualifiedIndexName = fmt.Sprintf("public.%s", fullyQualifiedIndexName) + } + result[fullyQualifiedIndexName] = true + } + return result, nil +} + +// TODO: need automation tests for this, covering cases like schema(public vs non-public) or case sensitive names +func beforeIndexCreation(sqlInfo sqlInfo, conn **pgx.Conn, objType string) error { + if !strings.Contains(strings.ToUpper(sqlInfo.stmt), "CREATE INDEX") { + return nil + } + + fullyQualifiedObjName, err := getIndexName(sqlInfo.stmt, sqlInfo.objName) + if err != nil { + return fmt.Errorf("extract qualified index name from DDL [%v]: %w", sqlInfo.stmt, err) + } + if invalidTargetIndexesCache == nil { + invalidTargetIndexesCache, err = getInvalidIndexes(conn) + if err != nil { + return fmt.Errorf("failed to fetch invalid indexes: %w", err) + } + } + + // check index valid or not + if invalidTargetIndexesCache[fullyQualifiedObjName] { + log.Infof("index %q already exists but in invalid state, dropping it", fullyQualifiedObjName) + err = dropIdx(*conn, fullyQualifiedObjName) + if err != nil { + return fmt.Errorf("drop invalid index %q: %w", fullyQualifiedObjName, err) + } + } + + // print the index name as index creation takes time and user can see the progress + color.Yellow("creating index %s ...", fullyQualifiedObjName) + return nil +} + +func dropIdx(conn *pgx.Conn, idxName string) error { + dropIdxQuery := fmt.Sprintf("DROP INDEX IF EXISTS %s", idxName) + log.Infof("Dropping index: %q", dropIdxQuery) + _, err := conn.Exec(context.Background(), dropIdxQuery) + if err != nil { + return fmt.Errorf("failed to drop index %q: %w", idxName, err) + } + return nil +} + +func newTargetConn() *pgx.Conn { + // save notice in global variable + noticeHandler := func(conn *pgconn.PgConn, n *pgconn.Notice) { + // ALTER TABLE .. ADD PRIMARY KEY throws the following notice in YugabyteDB. + // unlogged=# ALTER TABLE ONLY public.ul ADD CONSTRAINT ul_pkey PRIMARY KEY (id); + // NOTICE: table rewrite may lead to inconsistencies + // DETAIL: Concurrent DMLs may not be reflected in the new table. + // HINT: See https://github.com/yugabyte/yugabyte-db/issues/19860. Set 'ysql_suppress_unsafe_alter_notice' yb-tserver gflag to true to suppress this notice. + + // We ignore this notice because: + // 1. This is an empty table at the time at which we are importing the schema + // and there is no concurrent DMLs + // 2. This would unnecessarily clutter the output with NOTICES for every table, + // and scare the user + noticesToIgnore := []string{ + "table rewrite may lead to inconsistencies", + } + + if n != nil { + if lo.Contains(noticesToIgnore, n.Message) { + notice = nil + return + } + } + notice = n + } + errExit := func(err error) { + if err != nil { + utils.WaitChannel <- 1 + <-utils.WaitChannel + utils.ErrExit("connect to target db: %s", err) + } + } + + conf, err := pgx.ParseConfig(tconf.GetConnectionUri()) + errExit(err) + conf.OnNotice = noticeHandler + + conn, err := pgx.ConnectConfig(context.Background(), conf) + errExit(err) + + setTargetSchema(conn) + + if sourceDBType == ORACLE && enableOrafce { + setOrafceSearchPath(conn) + } + + return conn +} + +func getNoticeMessage(n *pgconn.Notice) string { + if n == nil { + return "" + } + return fmt.Sprintf("%s: %s", n.Severity, n.Message) +} + +// TODO: Eventually get rid of this function in favour of TargetYugabyteDB.setTargetSchema(). +func setTargetSchema(conn *pgx.Conn) { + if sourceDBType == POSTGRESQL || tconf.Schema == YUGABYTEDB_DEFAULT_SCHEMA { + // For PG, schema name is already included in the object name. + // No need to set schema if importing in the default schema. + return + } + checkSchemaExistsQuery := fmt.Sprintf("SELECT count(schema_name) FROM information_schema.schemata WHERE schema_name = '%s'", tconf.Schema) + var cntSchemaName int + + if err := conn.QueryRow(context.Background(), checkSchemaExistsQuery).Scan(&cntSchemaName); err != nil { + utils.ErrExit("run query %q on target %q to check schema exists: %s", checkSchemaExistsQuery, tconf.Host, err) + } else if cntSchemaName == 0 { + utils.ErrExit("schema '%s' does not exist in target", tconf.Schema) + } + + setSchemaQuery := fmt.Sprintf("SET SCHEMA '%s'", tconf.Schema) + _, err := conn.Exec(context.Background(), setSchemaQuery) + if err != nil { + utils.ErrExit("run query %q on target %q: %s", setSchemaQuery, tconf.Host, err) + } +} + +func setOrafceSearchPath(conn *pgx.Conn) { + // append oracle schema in the search_path for orafce + updateSearchPath := `SELECT set_config('search_path', current_setting('search_path') || ', oracle', false)` + _, err := conn.Exec(context.Background(), updateSearchPath) + if err != nil { + utils.ErrExit("unable to update search_path for orafce extension: %v", err) + } +} + +func execStmtAndGetNotice(conn *pgx.Conn, stmt string) (*pgconn.Notice, error) { + notice = nil // reset notice. + _, err := conn.Exec(context.Background(), stmt) + return notice, err +} diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 390cdf31dd..08fe2c69c1 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -44,6 +44,9 @@ var unloggedTableIssue = issue.Issue{ GH: "https://github.com/yugabyte/yugabyte-db/issues/1129/", Suggestion: "Remove UNLOGGED keyword to make it work", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", + MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ + ybversion.SERIES_2024_2: ybversion.V2024_2_0_0, + }, } func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement string) QueryIssue { diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index bc9beebf27..887b2759a1 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" @@ -73,32 +72,6 @@ func assertErrorCorrectlyThrownForIssueForYBVersion(t *testing.T, execErr error, } } -func getConnWithNoticeHandler(noticeHandler func(*pgconn.PgConn, *pgconn.Notice)) (*pgx.Conn, error) { - ctx := context.Background() - var connStr string - var err error - if testYugabytedbConnStr != "" { - connStr = testYugabytedbConnStr - } else { - connStr, err = testYugabytedbContainer.YSQLConnectionString(ctx, "sslmode=disable") - if err != nil { - return nil, err - } - } - - conf, err := pgx.ParseConfig(connStr) - if err != nil { - return nil, err - } - conf.OnNotice = noticeHandler - conn, err := pgx.ConnectConfig(ctx, conf) - if err != nil { - return nil, err - } - - return conn, nil -} - func testXMLFunctionIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -126,26 +99,14 @@ func testStoredGeneratedFunctionsIssue(t *testing.T) { } func testUnloggedTableIssue(t *testing.T) { - noticeFound := false - noticeHandler := func(conn *pgconn.PgConn, notice *pgconn.Notice) { - if notice != nil && notice.Message != "" { - assert.Equal(t, "unlogged option is currently ignored in YugabyteDB, all non-temp tables will be logged", notice.Message) - noticeFound = true - } - } ctx := context.Background() - conn, err := getConnWithNoticeHandler(noticeHandler) + conn, err := getConn() assert.NoError(t, err) defer conn.Close(context.Background()) _, err = conn.Exec(ctx, "CREATE UNLOGGED TABLE unlogged_table (a int)") - // in 2024.2, UNLOGGED no longer throws an error, just a notice - if noticeFound { - return - } else { - assert.ErrorContains(t, err, "UNLOGGED database object not supported yet") - } + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "UNLOGGED database object not supported yet", unloggedTableIssue) } func testAlterTableAddPKOnPartitionIssue(t *testing.T) { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 2fc872014b..e670ccc679 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -59,6 +59,10 @@ type ParserIssueDetector struct { //Boolean to check if there are any Gin indexes IsGinIndexPresentInSchema bool + + // Boolean to check if there are any unlogged tables that were filtered + // out because they are fixed as per the target db version + IsUnloggedTablesIssueFiltered bool } func NewParserIssueDetector() *ParserIssueDetector { @@ -113,6 +117,10 @@ func (p *ParserIssueDetector) getIssuesNotFixedInTargetDbVersion(issues []QueryI } if !fixed { filteredIssues = append(filteredIssues, i) + } else { + if i.Issue.Type == UNLOGGED_TABLE { + p.IsUnloggedTablesIssueFiltered = true + } } } return filteredIssues, nil diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e9330a09e4..cbb2775b22 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -180,7 +180,6 @@ func TestAllIssues(t *testing.T) { stmt2: []QueryIssue{ NewPercentTypeSyntaxIssue("FUNCTION", "process_order", "orders.id%TYPE"), NewStorageParameterIssue("TABLE", "public.example", "ALTER TABLE ONLY public.example ADD CONSTRAINT example_email_key UNIQUE (email) WITH (fillfactor=70);"), - NewUnloggedTableIssue("TABLE", "tbl_unlog", "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);"), NewMultiColumnGinIndexIssue("INDEX", "idx_example ON example_table", "CREATE INDEX idx_example ON example_table USING gin(name, name1);"), NewUnsupportedIndexMethodIssue("INDEX", "idx_example ON schema1.example_table", "CREATE INDEX idx_example ON schema1.example_table USING gist(name);", "gist"), NewAdvisoryLocksIssue("DML_QUERY", "", "SELECT pg_advisory_unlock(orderid);"), @@ -289,3 +288,19 @@ func TestDDLIssues(t *testing.T) { } } } + +func TestUnloggedTableIssueReportedInOlderVersion(t *testing.T) { + stmt := "CREATE UNLOGGED TABLE tbl_unlog (id int, val text);" + parserIssueDetector := NewParserIssueDetector() + + // Not reported by default + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + fatalIfError(t, err) + assert.Equal(t, 0, len(issues)) + + // older version should report the issue + issues, err = parserIssueDetector.GetDDLIssues(stmt, ybversion.V2024_1_0_0) + fatalIfError(t, err) + assert.Equal(t, 1, len(issues)) + assert.True(t, cmp.Equal(issues[0], NewUnloggedTableIssue("TABLE", "tbl_unlog", stmt))) +} diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index 1016a51d66..01d21b545d 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -32,6 +32,7 @@ import ( "syscall" "time" + "github.com/fatih/color" "github.com/google/uuid" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -445,11 +446,15 @@ func GetSqlStmtToPrint(stmt string) string { } } -func PrintSqlStmtIfDDL(stmt string, fileName string) { +func PrintSqlStmtIfDDL(stmt string, fileName string, noticeMsg string) { setOrSelectStmt := strings.HasPrefix(strings.ToUpper(stmt), "SET ") || strings.HasPrefix(strings.ToUpper(stmt), "SELECT ") if !setOrSelectStmt { fmt.Printf("%s: %s\n", fileName, GetSqlStmtToPrint(stmt)) + if noticeMsg != "" { + fmt.Printf(color.YellowString("%s\n", noticeMsg)) + log.Infof("notice for %q: %s", GetSqlStmtToPrint(stmt), noticeMsg) + } } } From d60b3ffc198f44636cd5dd68f1d489d184cd4c54 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:37:37 +0530 Subject: [PATCH 057/105] Printing a note if the voyager version does not match with the export dir version (#2075) --- yb-voyager/cmd/common.go | 29 +++++++++++++++++++++-------- yb-voyager/cmd/root.go | 8 ++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index dd6d459da7..468debe2da 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -480,14 +480,6 @@ func initMetaDB(migrationExportDir string) *metadb.MetaDB { utils.ErrExit("could not init migration status record: %w", err) } - msr, err := metaDBInstance.GetMigrationStatusRecord() - if err != nil { - utils.ErrExit("get migration status record: %v", err) - } - - msrVoyagerVersionString := msr.VoyagerVersion - - detectVersionCompatibility(msrVoyagerVersionString, migrationExportDir) return metaDBInstance } @@ -522,6 +514,27 @@ func detectVersionCompatibility(msrVoyagerVersionString string, migrationExportD if msrVoyagerVersion.LessThan(previousBreakingChangeVersion) { versionCheckFailed = true + } else { + // If the export-dir was created using a version greater than or equal to the PREVIOUS_BREAKING_CHANGE_VERSION, + // then if the current voyager version does not match the export-dir version, then just print a note warning the user. + noteString := fmt.Sprintf(color.YellowString("Note: The export-dir %q was created using voyager version %q. "+ + "The current version is %q."), + migrationExportDir, msrVoyagerVersionString, utils.YB_VOYAGER_VERSION) + + if utils.YB_VOYAGER_VERSION == "main" { + // In this case we won't be able to convert the version using version.NewVersion() as "main" is not a valid version. + // Moreover, we know here that the msrVoyagerVersion is not "main" as we have already handled that case above. + // Therefore, the current version and the msrVoyagerVersion will not be equal. + utils.PrintAndLog("%s", noteString) + } else { + currentVersion, err := version.NewVersion(utils.YB_VOYAGER_VERSION) + if err != nil { + utils.ErrExit("could not create version from %q: %v", utils.YB_VOYAGER_VERSION, err) + } + if !currentVersion.Equal(msrVoyagerVersion) { + utils.PrintAndLog("%s", noteString) + } + } } } diff --git a/yb-voyager/cmd/root.go b/yb-voyager/cmd/root.go index 602f1613b8..c9e7f87cca 100644 --- a/yb-voyager/cmd/root.go +++ b/yb-voyager/cmd/root.go @@ -128,6 +128,14 @@ Refer to docs (https://docs.yugabyte.com/preview/migrate/) for more details like // Initialize the metaDB variable only if the metaDB is already created. For example, resumption of a command. if metaDBIsCreated(exportDir) { metaDB = initMetaDB(exportDir) + msr, err := metaDB.GetMigrationStatusRecord() + if err != nil { + utils.ErrExit("get migration status record: %v", err) + } + + msrVoyagerVersionString := msr.VoyagerVersion + + detectVersionCompatibility(msrVoyagerVersionString, exportDir) } if perfProfile { From 414d4643433c7cf328fae117b359e79d1be8ed36 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Tue, 17 Dec 2024 15:32:17 +0530 Subject: [PATCH 058/105] Add --truncate-tables for import-data-file and import-data-to-source-replica (#2091) Add truncate-tables flag for import-data-file and import-data-to-source-replica. --- yb-voyager/cmd/importData.go | 8 ++---- yb-voyager/cmd/importDataFileCommand.go | 2 ++ yb-voyager/cmd/importDataToSourceReplica.go | 7 ++++-- yb-voyager/src/tgtdb/oracle.go | 28 ++++++++++++++++++++- yb-voyager/src/tgtdb/postgres.go | 7 ++++++ 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 3331df51b0..f70d489198 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -926,7 +926,7 @@ func cleanImportState(state *ImportDataState, tasks []*ImportFileTask) { nonEmptyTableNames := lo.Map(nonEmptyNts, func(nt sqlname.NameTuple, _ int) string { return nt.ForOutput() }) - if importerRole == TARGET_DB_IMPORTER_ROLE && truncateTables { + if truncateTables { // truncate tables only supported for import-data-to-target. utils.PrintAndLog("Truncating non-empty tables on DB: %v", nonEmptyTableNames) err := tdb.TruncateTables(nonEmptyNts) @@ -936,11 +936,7 @@ func cleanImportState(state *ImportDataState, tasks []*ImportFileTask) { } else { utils.PrintAndLog("Non-Empty tables: [%s]", strings.Join(nonEmptyTableNames, ", ")) utils.PrintAndLog("The above list of tables on DB are not empty.") - if importerRole == TARGET_DB_IMPORTER_ROLE { - utils.PrintAndLog("If you wish to truncate them, re-run the import command with --truncate-tables true") - } else { - utils.PrintAndLog("If you wish to truncate them, re-run the import command after manually truncating the tables on DB.") - } + utils.PrintAndLog("If you wish to truncate them, re-run the import command with --truncate-tables true") yes := utils.AskPrompt("Do you want to start afresh without truncating tables") if !yes { utils.ErrExit("Aborting import.") diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 5db7edf2aa..5985f4066c 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -489,6 +489,8 @@ If any table on YugabyteDB database is non-empty, it prompts whether you want to If you go ahead without truncating, then yb-voyager starts ingesting the data present in the data files with upsert mode. Note that for the cases where a table doesn't have a primary key, this may lead to insertion of duplicate data. To avoid this, exclude the table from --file-table-map or truncate those tables manually before using the start-clean flag (default false)`) + BoolVar(importDataFileCmd.Flags(), &truncateTables, "truncate-tables", false, "Truncate tables on target YugabyteDB before importing data. Only applicable along with --start-clean true (default false)") + importDataFileCmd.Flags().MarkHidden("table-list") importDataFileCmd.Flags().MarkHidden("exclude-table-list") importDataFileCmd.Flags().MarkHidden("table-list-file-path") diff --git a/yb-voyager/cmd/importDataToSourceReplica.go b/yb-voyager/cmd/importDataToSourceReplica.go index 5382fa2997..4c68550167 100644 --- a/yb-voyager/cmd/importDataToSourceReplica.go +++ b/yb-voyager/cmd/importDataToSourceReplica.go @@ -67,17 +67,20 @@ func init() { registerCommonImportFlags(importDataToSourceReplicaCmd) registerSourceReplicaDBAsTargetConnFlags(importDataToSourceReplicaCmd) registerFlagsForSourceReplica(importDataToSourceReplicaCmd) - registerStartCleanFlag(importDataToSourceReplicaCmd) + registerStartCleanFlags(importDataToSourceReplicaCmd) registerImportDataCommonFlags(importDataToSourceReplicaCmd) hideImportFlagsInFallForwardOrBackCmds(importDataToSourceReplicaCmd) } -func registerStartCleanFlag(cmd *cobra.Command) { +func registerStartCleanFlags(cmd *cobra.Command) { BoolVar(cmd.Flags(), &startClean, "start-clean", false, `Starts a fresh import with exported data files present in the export-dir/data directory. If any table on source-replica database is non-empty, it prompts whether you want to continue the import without truncating those tables; If you go ahead without truncating, then yb-voyager starts ingesting the data present in the data files without upsert mode. Note that for the cases where a table doesn't have a primary key, this may lead to insertion of duplicate data. To avoid this, exclude the table using the --exclude-file-list or truncate those tables manually before using the start-clean flag (default false)`) + + BoolVar(cmd.Flags(), &truncateTables, "truncate-tables", false, "Truncate tables on source replica DB before importing data. Only applicable along with --start-clean true (default false)") + } func updateFallForwardEnabledInMetaDB() { diff --git a/yb-voyager/src/tgtdb/oracle.go b/yb-voyager/src/tgtdb/oracle.go index b66fbc986c..759bd7cbc0 100644 --- a/yb-voyager/src/tgtdb/oracle.go +++ b/yb-voyager/src/tgtdb/oracle.go @@ -29,6 +29,7 @@ import ( "time" "github.com/google/uuid" + "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" @@ -192,8 +193,33 @@ func (tdb *TargetOracleDB) GetNonEmptyTables(tables []sqlname.NameTuple) []sqlna return result } +// There are some restrictions on TRUNCATE TABLE in oracle +// https://docs.oracle.com/en/database/oracle/oracle-database/23/sqlrf/TRUNCATE-TABLE.html#:~:text=cursors%20are%20invalidated.-,Restrictions%20on%20Truncating%20Tables,-This%20statement%20is +// Notable ones include: +// 1. You cannot truncate a table that has a foreign key constraint. This is not a problem because in our fall-forward workflow, +// we ask users to disable foreign key constraints before starting the migration. +// 2. You cannot truncate a table that is part of a cluster. This will fail. +// 3. You cannot truncate the parent table of a reference-partitioned table. This will fail. voyager does not support reference partitioned tables migration. +// +// Given that there are some cases where it might fail, we attempt to truncate all tables wherever possible, +// and accumulate all the errors and return them as a single error. func (tdb *TargetOracleDB) TruncateTables(tables []sqlname.NameTuple) error { - return fmt.Errorf("truncate tables not implemented for oracle") + tableNames := lo.Map(tables, func(nt sqlname.NameTuple, _ int) string { + return nt.ForUserQuery() + }) + var errors []error + + for _, tableName := range tableNames { + query := fmt.Sprintf("TRUNCATE TABLE %s", tableName) + _, err := tdb.Exec(query) + if err != nil { + errors = append(errors, fmt.Errorf("truncate table %q: %w", tableName, err)) + } + } + if len(errors) > 0 { + return fmt.Errorf("truncate tables: %v", errors) + } + return nil } func (tdb *TargetOracleDB) IsNonRetryableCopyError(err error) bool { diff --git a/yb-voyager/src/tgtdb/postgres.go b/yb-voyager/src/tgtdb/postgres.go index 92ac7ec1be..b9289aac4f 100644 --- a/yb-voyager/src/tgtdb/postgres.go +++ b/yb-voyager/src/tgtdb/postgres.go @@ -29,6 +29,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" + pgconn5 "github.com/jackc/pgx/v5/pgconn" _ "github.com/jackc/pgx/v5/stdlib" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -76,6 +77,12 @@ func (pg *TargetPostgreSQL) Exec(query string) (int64, error) { res, err := pg.db.Exec(query) if err != nil { + var pgErr *pgconn5.PgError + if errors.As(err, &pgErr) { + if pgErr.Hint != "" || pgErr.Detail != "" { + return rowsAffected, fmt.Errorf("run query %q on target %q: %w \nHINT: %s\nDETAIL: %s", query, pg.tconf.Host, err, pgErr.Hint, pgErr.Detail) + } + } return rowsAffected, fmt.Errorf("run query %q on target %q: %w", query, pg.tconf.Host, err) } rowsAffected, err = res.RowsAffected() From d22fbdc748368328b122d1417fd7d114a1a204ff Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Tue, 17 Dec 2024 15:40:33 +0530 Subject: [PATCH 059/105] Redacting password while logging connection params in import data to target (#2097) --- yb-voyager/src/tgtdb/yugabytedb.go | 4 +++- yb-voyager/src/utils/utils.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index baa39102ee..5d20b0ebb0 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -242,7 +242,9 @@ func (yb *TargetYugabyteDB) InitConnPool() error { SessionInitScript: getYBSessionInitScript(yb.tconf), } yb.connPool = NewConnectionPool(params) - log.Info("Initialized connection pool with settings: ", spew.Sdump(params)) + redactedParams := params + redactedParams.ConnUriList = utils.GetRedactedURLs(redactedParams.ConnUriList) + log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil } diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index 01d21b545d..c960814b9b 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -431,7 +431,8 @@ func GetRedactedURLs(urlList []string) []string { for _, u := range urlList { obj, err := url.Parse(u) if err != nil { - ErrExit("invalid URL: %q", u) + log.Error("error redacting connection url: invalid connection URL") + fmt.Printf("error redacting connection url: invalid connection URL: %v", u) } result = append(result, obj.Redacted()) } From 502f338b5f515dd212c8ed19d01a691379623195 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Tue, 17 Dec 2024 18:14:56 +0530 Subject: [PATCH 060/105] Misc changes to target-db-version flag (#2098) - Unhide the flag - Print the version being used in analyze-schema/ assess-migration - Prevent users from passing build number in target-db-version. The problem is that 2024.2.0.0-b145 would be considered to be > 2024.2.0.0. Which means we will have two options: a. prevent users from passing build number b. Always ask users to pass build number, so that it can be compared against the issues' MinimumVersionsFixedIn(which should also have build number.) Going with option a. --- yb-voyager/cmd/analyzeSchema.go | 4 ++-- yb-voyager/cmd/assessMigrationCommand.go | 6 +++--- yb-voyager/src/ybversion/yb_version.go | 7 ++++++- yb-voyager/src/ybversion/yb_version_test.go | 13 +++++++------ 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 1e30245a2d..c569bd9c0f 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1082,6 +1082,7 @@ func analyzeSchema() { utils.ErrExit("failed to get migration UUID: %w", err) } + utils.PrintAndLog("Analyzing schema for target YugabyteDB version %s\n", targetDbVersion) schemaAnalysisStartedEvent := createSchemaAnalysisStartedEvent() controlPlane.SchemaAnalysisStarted(&schemaAnalysisStartedEvent) @@ -1260,8 +1261,7 @@ func init() { "format in which report can be generated: ('html', 'txt', 'json', 'xml'). If not provided, reports will be generated in both 'json' and 'html' formats by default.") analyzeSchemaCmd.Flags().StringVar(&targetDbVersionStrFlag, "target-db-version", "", - fmt.Sprintf("Target YugabyteDB version to analyze schema for. Defaults to latest stable version (%s)", ybversion.LatestStable.String())) - analyzeSchemaCmd.Flags().MarkHidden("target-db-version") + fmt.Sprintf("Target YugabyteDB version to analyze schema for (in format A.B.C.D). Defaults to latest stable version (%s)", ybversion.LatestStable.String())) } func validateReportOutputFormat(validOutputFormats []string, format string) { diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 95270d6f45..767c5bf074 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -295,8 +295,7 @@ func init() { BoolVar(assessMigrationCmd.Flags(), &source.RunGuardrailsChecks, "run-guardrails-checks", true, "run guardrails checks before assess migration. (only valid for PostgreSQL)") assessMigrationCmd.Flags().StringVar(&targetDbVersionStrFlag, "target-db-version", "", - fmt.Sprintf("Target YugabyteDB version to assess migration for. Defaults to latest stable version (%s)", ybversion.LatestStable.String())) - assessMigrationCmd.Flags().MarkHidden("target-db-version") + fmt.Sprintf("Target YugabyteDB version to assess migration for (in format A.B.C.D). Defaults to latest stable version (%s)", ybversion.LatestStable.String())) } func assessMigration() (err error) { @@ -313,6 +312,8 @@ func assessMigration() (err error) { return fmt.Errorf("failed to get migration UUID: %w", err) } + utils.PrintAndLog("Assessing for migration to target YugabyteDB version %s\n", targetDbVersion) + assessmentDir := filepath.Join(exportDir, "assessment") migassessment.AssessmentDir = assessmentDir migassessment.SourceDBType = source.DBType @@ -1546,7 +1547,6 @@ func validateAssessmentMetadataDirFlag() { func validateAndSetTargetDbVersionFlag() error { if targetDbVersionStrFlag == "" { targetDbVersion = ybversion.LatestStable - utils.PrintAndLog("Defaulting to latest stable YugabyteDB version: %s", targetDbVersion) return nil } var err error diff --git a/yb-voyager/src/ybversion/yb_version.go b/yb-voyager/src/ybversion/yb_version.go index b8d2264587..fad15e0863 100644 --- a/yb-voyager/src/ybversion/yb_version.go +++ b/yb-voyager/src/ybversion/yb_version.go @@ -43,7 +43,7 @@ const ( /* YBVersion is a wrapper around hashicorp/go-version.Version that adds some Yugabyte-specific functionality. - 1. It only supports versions with 4 segments (A.B.C.D). + 1. It only supports versions with 4 segments (A.B.C.D). No build number is allowed. 2. It only accepts one of supported Yugabyte version series. */ type YBVersion struct { @@ -56,6 +56,11 @@ func NewYBVersion(v string) (*YBVersion, error) { return nil, err } + // Do not allow build number in the version. for example, 2024.1.1.0-b123 + if v1.Prerelease() != "" { + return nil, fmt.Errorf("invalid YB version: %s. Build number is not supported. Version should be of format (A.B.C.D) ", v) + } + ybv := &YBVersion{v1} origSegLen := ybv.OriginalSegmentsLen() if origSegLen != 4 { diff --git a/yb-voyager/src/ybversion/yb_version_test.go b/yb-voyager/src/ybversion/yb_version_test.go index b29711bad9..0e53526751 100644 --- a/yb-voyager/src/ybversion/yb_version_test.go +++ b/yb-voyager/src/ybversion/yb_version_test.go @@ -40,15 +40,16 @@ func TestValidNewYBVersion(t *testing.T) { func TestInvalidNewYBVersion(t *testing.T) { invalidVersionStrings := []string{ - "abc.def", // has to be numbers - "2024.0.1-1", // has to be in supported series - "2024", // has to have 4 segments - "2.20.7", // has to have 4 segments - "2024.1.1.1.1.1", // exactly 4 segments + "abc.def", // has to be numbers + "2024.0.1-1", // has to be in supported series + "2024", // has to have 4 segments + "2.20.7", // has to have 4 segments + "2024.1.1.1.1.1", // exactly 4 segments + "2024.1.0.0-b123", // build number is not allowed. } for _, v := range invalidVersionStrings { _, err := NewYBVersion(v) - assert.Error(t, err) + assert.Errorf(t, err, "expected error for %q", v) } } From 563473b39f2e18c4f9533e60283ecb121bff476b Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Wed, 18 Dec 2024 13:07:39 +0530 Subject: [PATCH 061/105] Minor refactor to proto detectors (#2094) - Change interface of UnsupportedConstructDetector to make each detector responsible for constructing and returning issues. Currently, this was being done in ParserIssueDetector.getGenericIssues, which seems like the wrong place for it. With pg17 features, all the issue construction will be added to this function itself, which I want to avoid. Detectors do the detection, and they should also construct the issues since they have all the information necessary to do so. - Store functions being detected in FuncCallDetector . Will be useful for later reporting. - This refactor led to one side-effect related to XML issues. Worked around that for now. --- yb-voyager/go.mod | 1 + yb-voyager/go.sum | 2 + yb-voyager/src/query/queryissue/detectors.go | 136 +++++---- .../src/query/queryissue/detectors_test.go | 264 ++++++++++-------- yb-voyager/src/query/queryissue/helpers.go | 16 +- .../query/queryissue/parser_issue_detector.go | 39 ++- .../queryissue/parser_issue_detector_test.go | 20 ++ 7 files changed, 295 insertions(+), 183 deletions(-) diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 93e64b1e7f..c833375e2f 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -55,6 +55,7 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/deckarep/golang-set/v2 v2.7.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 1369ff42c2..44ee2af8de 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -836,6 +836,8 @@ github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjI github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k= +github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index a89da8c544..680c8b7254 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,10 +16,10 @@ limitations under the License. package queryissue import ( + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/reflect/protoreflect" - "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" + "google.golang.org/protobuf/reflect/protoreflect" ) const ( @@ -31,86 +31,115 @@ const ( // To Add a new unsupported query construct implement this interface for all possible nodes for that construct // each detector will work on specific type of node type UnsupportedConstructDetector interface { - Detect(msg protoreflect.Message) ([]string, error) + Detect(msg protoreflect.Message) error + GetIssues() []QueryIssue } type FuncCallDetector struct { + query string // right now it covers Advisory Locks and XML functions - unsupportedFuncs map[string]string + advisoryLocksFuncsDetected mapset.Set[string] + xmlFuncsDetected mapset.Set[string] } -func NewFuncCallDetector() *FuncCallDetector { - unsupportedFuncs := make(map[string]string) - for _, fname := range unsupportedAdvLockFuncs { - unsupportedFuncs[fname] = ADVISORY_LOCKS_NAME - } - for _, fname := range unsupportedXmlFunctions { - unsupportedFuncs[fname] = XML_FUNCTIONS_NAME - } - +func NewFuncCallDetector(query string) *FuncCallDetector { return &FuncCallDetector{ - unsupportedFuncs: unsupportedFuncs, + query: query, + advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](), + xmlFuncsDetected: mapset.NewThreadUnsafeSet[string](), } } // Detect checks if a FuncCall node uses an unsupported function. -func (d *FuncCallDetector) Detect(msg protoreflect.Message) ([]string, error) { +func (d *FuncCallDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_FUNCCALL_NODE { - return nil, nil + return nil } _, funcName := queryparser.GetFuncNameFromFuncCall(msg) log.Debugf("fetched function name from %s node: %q", queryparser.PG_QUERY_FUNCCALL_NODE, funcName) - if constructType, isUnsupported := d.unsupportedFuncs[funcName]; isUnsupported { - log.Debugf("detected unsupported function %q in msg - %+v", funcName, msg) - return []string{constructType}, nil + + if unsupportedAdvLockFuncs.ContainsOne(funcName) { + d.advisoryLocksFuncsDetected.Add(funcName) + } + if unsupportedXmlFunctions.ContainsOne(funcName) { + d.xmlFuncsDetected.Add(funcName) } - return nil, nil -} -type ColumnRefDetector struct { - unsupportedColumns map[string]string + return nil } -func NewColumnRefDetector() *ColumnRefDetector { - unsupportedColumns := make(map[string]string) - for _, colName := range unsupportedSysCols { - unsupportedColumns[colName] = SYSTEM_COLUMNS_NAME +func (d *FuncCallDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.advisoryLocksFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewAdvisoryLocksIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } + if d.xmlFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + +type ColumnRefDetector struct { + query string + unsupportedSystemColumnsDetected mapset.Set[string] +} +func NewColumnRefDetector(query string) *ColumnRefDetector { return &ColumnRefDetector{ - unsupportedColumns: unsupportedColumns, + query: query, + unsupportedSystemColumnsDetected: mapset.NewThreadUnsafeSet[string](), } } // Detect checks if a ColumnRef node uses an unsupported system column -func (d *ColumnRefDetector) Detect(msg protoreflect.Message) ([]string, error) { +func (d *ColumnRefDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_COLUMNREF_NODE { - return nil, nil + return nil } _, colName := queryparser.GetColNameFromColumnRef(msg) log.Debugf("fetched column name from %s node: %q", queryparser.PG_QUERY_COLUMNREF_NODE, colName) - if constructType, isUnsupported := d.unsupportedColumns[colName]; isUnsupported { - log.Debugf("detected unsupported system column %q in msg - %+v", colName, msg) - return []string{constructType}, nil + + if unsupportedSysCols.ContainsOne(colName) { + d.unsupportedSystemColumnsDetected.Add(colName) } - return nil, nil + return nil } -type XmlExprDetector struct{} +func (d *ColumnRefDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.unsupportedSystemColumnsDetected.Cardinality() > 0 { + issues = append(issues, NewSystemColumnsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} -func NewXmlExprDetector() *XmlExprDetector { - return &XmlExprDetector{} +type XmlExprDetector struct { + query string + detected bool +} + +func NewXmlExprDetector(query string) *XmlExprDetector { + return &XmlExprDetector{ + query: query, + } } // Detect checks if a XmlExpr node is present, means Xml type/functions are used -func (d *XmlExprDetector) Detect(msg protoreflect.Message) ([]string, error) { +func (d *XmlExprDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_XMLEXPR_NODE { - log.Debug("detected xml expression") - return []string{XML_FUNCTIONS_NAME}, nil + d.detected = true + } + return nil +} + +func (d *XmlExprDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.detected { + issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } - return nil, nil + return issues } /* @@ -126,18 +155,31 @@ ASSUMPTION: - link: https://github.com/postgres/postgres/blob/ea792bfd93ab8ad4ef4e3d1a741b8595db143677/src/include/nodes/parsenodes.h#L651 */ -type RangeTableFuncDetector struct{} +type RangeTableFuncDetector struct { + query string + detected bool +} -func NewRangeTableFuncDetector() *RangeTableFuncDetector { - return &RangeTableFuncDetector{} +func NewRangeTableFuncDetector(query string) *RangeTableFuncDetector { + return &RangeTableFuncDetector{ + query: query, + } } // Detect checks if a RangeTableFunc node is present for a XMLTABLE() function -func (d *RangeTableFuncDetector) Detect(msg protoreflect.Message) ([]string, error) { +func (d *RangeTableFuncDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_RANGETABLEFUNC_NODE { if queryparser.IsXMLTable(msg) { - return []string{XML_FUNCTIONS_NAME}, nil + d.detected = true } } - return nil, nil + return nil +} + +func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.detected { + issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues } diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 194aff8e4e..2845fc2adb 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -19,7 +19,7 @@ import ( "fmt" "testing" - "github.com/samber/lo" + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/reflect/protoreflect" @@ -76,27 +76,30 @@ func TestFuncCallDetector(t *testing.T) { `SELECT pg_advisory_unlock_all();`, } - detector := NewFuncCallDetector() for _, sql := range advisoryLockSqls { + detector := NewFuncCallDetector(sql) + parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} processor := func(msg protoreflect.Message) error { - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { return err } - unsupportedConstructs = append(unsupportedConstructs, constructs...) return nil } parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.Contains(t, unsupportedConstructs, ADVISORY_LOCKS_NAME, "Advisory Locks not detected in SQL: %s", sql) + + issues := detector.GetIssues() + + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, ADVISORY_LOCKS, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } } @@ -141,27 +144,29 @@ func TestColumnRefDetector(t *testing.T) { HAVING COUNT(*) > 1;`, } - detector := NewColumnRefDetector() for _, sql := range systemColumnSqls { + detector := NewColumnRefDetector(sql) parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} processor := func(msg protoreflect.Message) error { - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { return err } - unsupportedConstructs = append(unsupportedConstructs, constructs...) return nil } parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.Contains(t, unsupportedConstructs, SYSTEM_COLUMNS_NAME, "System Columns not detected in SQL: %s", sql) + + issues := detector.GetIssues() + + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, SYSTEM_COLUMNS, issues[0].Type, "Expected System Columns issue for SQL: %s", sql) } } @@ -325,27 +330,29 @@ func TestRangeTableFuncDetector(t *testing.T) { s.report_id;`, } - detector := NewRangeTableFuncDetector() for _, sql := range xmlTableSqls { + detector := NewRangeTableFuncDetector(sql) parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} processor := func(msg protoreflect.Message) error { - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { return err } - unsupportedConstructs = append(unsupportedConstructs, constructs...) return nil } parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS_NAME, "XML Functions not detected in SQL: %s", sql) + + issues := detector.GetIssues() + + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, XML_FUNCTIONS, issues[0].Type, "Expected XML Functions issue for SQL: %s", sql) } } @@ -356,85 +363,85 @@ func TestXMLFunctionsDetectors(t *testing.T) { `SELECT id, xpath('/person/name/text()', data) AS name FROM xml_example;`, `SELECT id FROM employees WHERE xmlexists('/id' PASSING BY VALUE xmlcolumn);`, `SELECT e.id, x.employee_xml - FROM employees e - JOIN ( - SELECT xmlelement(name "employee", xmlattributes(e.id AS "id"), e.name) AS employee_xml - FROM employees e - ) x ON x.employee_xml IS NOT NULL - WHERE xmlexists('//employee[name="John Doe"]' PASSING BY REF x.employee_xml);`, + FROM employees e + JOIN ( + SELECT xmlelement(name "employee", xmlattributes(e.id AS "id"), e.name) AS employee_xml + FROM employees e + ) x ON x.employee_xml IS NOT NULL + WHERE xmlexists('//employee[name="John Doe"]' PASSING BY REF x.employee_xml);`, `WITH xml_data AS ( - SELECT - id, - xml_column, - xpath('/root/element/@attribute', xml_column) as xpath_result - FROM xml_documents - ) - SELECT - x.id, - (xt.value).text as value - FROM - xml_data x - CROSS JOIN LATERAL unnest(x.xpath_result) as xt(value);`, + SELECT + id, + xml_column, + xpath('/root/element/@attribute', xml_column) as xpath_result + FROM xml_documents + ) + SELECT + x.id, + (xt.value).text as value + FROM + xml_data x + CROSS JOIN LATERAL unnest(x.xpath_result) as xt(value);`, `SELECT e.id, e.name - FROM employees e - WHERE CASE - WHEN e.department = 'IT' THEN xmlexists('//access[@level="high"]' PASSING e.permissions) - ELSE FALSE - END;`, + FROM employees e + WHERE CASE + WHEN e.department = 'IT' THEN xmlexists('//access[@level="high"]' PASSING e.permissions) + ELSE FALSE + END;`, `SELECT xmlserialize( - content xmlelement(name "employees", - xmlagg( - xmlelement(name "employee", - xmlattributes(e.id AS "id"), - e.name - ) - ) - ) AS CLOB - ) AS employees_xml - FROM employees e - WHERE e.status = 'active';`, + content xmlelement(name "employees", + xmlagg( + xmlelement(name "employee", + xmlattributes(e.id AS "id"), + e.name + ) + ) + ) AS CLOB + ) AS employees_xml + FROM employees e + WHERE e.status = 'active';`, `CREATE VIEW employee_xml_view AS - SELECT e.id, - xmlelement(name "employee", - xmlattributes(e.id AS "id"), - e.name, - e.department - ) AS employee_xml - FROM employees e;`, + SELECT e.id, + xmlelement(name "employee", + xmlattributes(e.id AS "id"), + e.name, + e.department + ) AS employee_xml + FROM employees e;`, `SELECT xmltext('Widget') AS inventory_text -FROM inventory -WHERE id = 5;`, + FROM inventory + WHERE id = 5;`, `SELECT xmlforest(name, department) AS employee_info -FROM employees -WHERE id = 4;`, + FROM employees + WHERE id = 4;`, `SELECT xmltable.* - FROM xmldata, - XMLTABLE('//ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - ordinality FOR ORDINALITY, - "COUNTRY_NAME" text, - country_id text PATH 'COUNTRY_ID', - size_sq_km float PATH 'SIZE[@unit = "sq_km"]', - size_other text PATH - 'concat(SIZE[@unit!="sq_km"], " ", SIZE[@unit!="sq_km"]/@unit)', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified');`, + FROM xmldata, + XMLTABLE('//ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + ordinality FOR ORDINALITY, + "COUNTRY_NAME" text, + country_id text PATH 'COUNTRY_ID', + size_sq_km float PATH 'SIZE[@unit = "sq_km"]', + size_other text PATH + 'concat(SIZE[@unit!="sq_km"], " ", SIZE[@unit!="sq_km"]/@unit)', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified');`, `SELECT xmltable.* - FROM XMLTABLE(XMLNAMESPACES('http://example.com/myns' AS x, - 'http://example.com/b' AS "B"), - '/x:example/x:item' - PASSING (SELECT data FROM xmldata) - COLUMNS foo int PATH '@foo', - bar int PATH '@B:bar');`, + FROM XMLTABLE(XMLNAMESPACES('http://example.com/myns' AS x, + 'http://example.com/b' AS "B"), + '/x:example/x:item' + PASSING (SELECT data FROM xmldata) + COLUMNS foo int PATH '@foo', + bar int PATH '@B:bar');`, `SELECT xml_is_well_formed_content('Alpha') AS is_well_formed_content -FROM projects -WHERE project_id = 10;`, + FROM projects + WHERE project_id = 10;`, `SELECT xml_is_well_formed_document(xmlforest(name, department)) AS is_well_formed_document -FROM employees -WHERE id = 2;`, + FROM employees + WHERE id = 2;`, `SELECT xml_is_well_formed(xmltext('Jane Doe')) AS is_well_formed -FROM employees -WHERE id = 1;`, + FROM employees + WHERE id = 1;`, `SELECT xmlparse(DOCUMENT 'John');`, `SELECT xpath_exists('/employee/name', 'John'::xml)`, `SELECT table_to_xml('employees', TRUE, FALSE, '');`, @@ -465,28 +472,26 @@ WHERE id = 1;`, `SELECT xml_send('send');`, } - detectors := []UnsupportedConstructDetector{ - NewXmlExprDetector(), - NewRangeTableFuncDetector(), - NewFuncCallDetector(), - } - for _, sql := range xmlFunctionSqls { + detectors := []UnsupportedConstructDetector{ + NewXmlExprDetector(sql), + NewRangeTableFuncDetector(sql), + NewFuncCallDetector(sql), + } + parseResult, err := queryparser.Parse(sql) assert.NoError(t, err) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} processor := func(msg protoreflect.Message) error { for _, detector := range detectors { log.Debugf("running detector %T", detector) - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { log.Debugf("error in detector %T: %v", detector, err) return fmt.Errorf("error in detectors %T: %w", detector, err) } - unsupportedConstructs = lo.Union(unsupportedConstructs, constructs) } return nil } @@ -494,8 +499,21 @@ WHERE id = 1;`, parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - // The detector should detect XML Functions in these queries - assert.Contains(t, unsupportedConstructs, XML_FUNCTIONS_NAME, "XML Functions not detected in SQL: %s", sql) + + var allIssues []QueryIssue + for _, detector := range detectors { + allIssues = append(allIssues, detector.GetIssues()...) + } + + xmlIssueDetected := false + for _, issue := range allIssues { + if issue.Type == XML_FUNCTIONS { + xmlIssueDetected = true + break + } + } + + assert.True(t, xmlIssueDetected, "Expected XML Functions issue for SQL: %s", sql) } } @@ -530,29 +548,27 @@ RETURNING id, xmlattributes(id AS "ID"), xmlforest(name AS "Name", salary AS "NewSalary", xmin AS "TransactionStartID", xmax AS "TransactionEndID"));`, } - expectedConstructs := []string{ADVISORY_LOCKS_NAME, SYSTEM_COLUMNS_NAME, XML_FUNCTIONS_NAME} + expectedIssueTypes := mapset.NewThreadUnsafeSet[string]([]string{ADVISORY_LOCKS, SYSTEM_COLUMNS, XML_FUNCTIONS}...) - detectors := []UnsupportedConstructDetector{ - NewFuncCallDetector(), - NewColumnRefDetector(), - NewXmlExprDetector(), - } for _, sql := range combinationSqls { + detectors := []UnsupportedConstructDetector{ + NewFuncCallDetector(sql), + NewColumnRefDetector(sql), + NewXmlExprDetector(sql), + } parseResult, err := queryparser.Parse(sql) assert.NoError(t, err) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} processor := func(msg protoreflect.Message) error { for _, detector := range detectors { log.Debugf("running detector %T", detector) - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { log.Debugf("error in detector %T: %v", detector, err) return fmt.Errorf("error in detectors %T: %w", detector, err) } - unsupportedConstructs = lo.Union(unsupportedConstructs, constructs) } return nil } @@ -560,7 +576,17 @@ RETURNING id, parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.ElementsMatch(t, expectedConstructs, unsupportedConstructs, "Detected constructs do not exactly match the expected constructs. Expected: %v, Actual: %v", expectedConstructs, unsupportedConstructs) + + var allIssues []QueryIssue + for _, detector := range detectors { + allIssues = append(allIssues, detector.GetIssues()...) + } + issueTypesDetected := mapset.NewThreadUnsafeSet[string]() + for _, issue := range allIssues { + issueTypesDetected.Add(issue.Type) + } + + assert.True(t, expectedIssueTypes.Equal(issueTypesDetected), "Expected issue types do not match the detected issue types. Expected: %v, Actual: %v", expectedIssueTypes, issueTypesDetected) } } @@ -613,30 +639,28 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { }, } - expectedConstructs := []string{ADVISORY_LOCKS_NAME, SYSTEM_COLUMNS_NAME, XML_FUNCTIONS_NAME} + expectedIssueTypes := mapset.NewThreadUnsafeSet[string]([]string{ADVISORY_LOCKS, SYSTEM_COLUMNS, XML_FUNCTIONS}...) - detectors := []UnsupportedConstructDetector{ - NewFuncCallDetector(), - NewColumnRefDetector(), - NewXmlExprDetector(), - } for _, tc := range tests { + detectors := []UnsupportedConstructDetector{ + NewFuncCallDetector(tc.Sql), + NewColumnRefDetector(tc.Sql), + NewXmlExprDetector(tc.Sql), + } parseResult, err := queryparser.Parse(tc.Sql) assert.NoError(t, err) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} objectCollector := queryparser.NewObjectCollector() processor := func(msg protoreflect.Message) error { for _, detector := range detectors { log.Debugf("running detector %T", detector) - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { log.Debugf("error in detector %T: %v", detector, err) return fmt.Errorf("error in detectors %T: %w", detector, err) } - unsupportedConstructs = lo.Union(unsupportedConstructs, constructs) } objectCollector.Collect(msg) return nil @@ -645,7 +669,17 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.ElementsMatch(t, expectedConstructs, unsupportedConstructs, "Detected constructs do not exactly match the expected constructs. Expected: %v, Actual: %v", expectedConstructs, unsupportedConstructs) + + var allIssues []QueryIssue + for _, detector := range detectors { + allIssues = append(allIssues, detector.GetIssues()...) + } + issueTypesDetected := mapset.NewThreadUnsafeSet[string]() + for _, issue := range allIssues { + issueTypesDetected.Add(issue.Type) + } + + assert.True(t, expectedIssueTypes.Equal(issueTypesDetected), "Expected issue types do not match the detected issue types. Expected: %v, Actual: %v", expectedIssueTypes, issueTypesDetected) collectedObjects := objectCollector.GetObjects() collectedSchemas := objectCollector.GetSchemaList() diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index fad34e2de0..41e3aa982a 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -16,21 +16,25 @@ limitations under the License. package queryissue +import ( + mapset "github.com/deckarep/golang-set/v2" +) + // Refer: https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS -var unsupportedAdvLockFuncs = []string{ +var unsupportedAdvLockFuncs = mapset.NewThreadUnsafeSet([]string{ "pg_advisory_lock", "pg_advisory_lock_shared", "pg_advisory_unlock", "pg_advisory_unlock_all", "pg_advisory_unlock_shared", "pg_advisory_xact_lock", "pg_advisory_xact_lock_shared", "pg_try_advisory_lock", "pg_try_advisory_lock_shared", "pg_try_advisory_xact_lock", "pg_try_advisory_xact_lock_shared", -} +}...) -var unsupportedSysCols = []string{ +var unsupportedSysCols = mapset.NewThreadUnsafeSet([]string{ "xmin", "xmax", "cmin", "cmax", "ctid", -} +}...) // Refer: https://www.postgresql.org/docs/17/functions-xml.html#FUNCTIONS-XML-PROCESSING -var unsupportedXmlFunctions = []string{ +var unsupportedXmlFunctions = mapset.NewThreadUnsafeSet([]string{ // 1. Producing XML content "xmltext", "xmlcomment", "xmlconcat", "xmlelement", "xmlforest", "xmlpi", "xmlroot", "xmlagg", @@ -52,7 +56,7 @@ var unsupportedXmlFunctions = []string{ WHERE prorettype = 'xml'::regtype; */ "xmlconcat2", "xmlvalidate", "xml_in", "xml_out", "xml_recv", "xml_send", // System XML I/O -} +}...) var UnsupportedIndexMethods = []string{ "gist", diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index e670ccc679..ffa1d98605 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -369,24 +369,22 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) return nil, fmt.Errorf("error parsing query: %w", err) } var result []QueryIssue - var unsupportedConstructs []string visited := make(map[protoreflect.Message]bool) detectors := []UnsupportedConstructDetector{ - NewFuncCallDetector(), - NewColumnRefDetector(), - NewXmlExprDetector(), - NewRangeTableFuncDetector(), + NewFuncCallDetector(query), + NewColumnRefDetector(query), + NewXmlExprDetector(query), + NewRangeTableFuncDetector(query), } processor := func(msg protoreflect.Message) error { for _, detector := range detectors { log.Debugf("running detector %T", detector) - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { log.Debugf("error in detector %T: %v", detector, err) return fmt.Errorf("error in detectors %T: %w", detector, err) } - unsupportedConstructs = lo.Union(unsupportedConstructs, constructs) } return nil } @@ -397,15 +395,26 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) return result, fmt.Errorf("error traversing parse tree message: %w", err) } - for _, unsupportedConstruct := range unsupportedConstructs { - switch unsupportedConstruct { - case ADVISORY_LOCKS_NAME: - result = append(result, NewAdvisoryLocksIssue(DML_QUERY_OBJECT_TYPE, "", query)) - case SYSTEM_COLUMNS_NAME: - result = append(result, NewSystemColumnsIssue(DML_QUERY_OBJECT_TYPE, "", query)) - case XML_FUNCTIONS_NAME: - result = append(result, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", query)) + xmlIssueAdded := false + for _, detector := range detectors { + issues := detector.GetIssues() + for _, issue := range issues { + if issue.Type == XML_FUNCTIONS { + if xmlIssueAdded { + // currently, both FuncCallDetector and XmlExprDetector can detect XMLFunctionsIssue + // but we want to only return one XMLFunctionsIssue. + // TODO: refactor to avoid this + // Possible Solutions: + // 1. Have a dedicated detector for XMLFunctions and Expressions so that a single issue is returned + // 2. Separate issue types for XML Functions and XML expressions. + continue + } else { + xmlIssueAdded = true + } + } + result = append(result, issues...) } } + return result, nil } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index cbb2775b22..99d989b528 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -304,3 +304,23 @@ func TestUnloggedTableIssueReportedInOlderVersion(t *testing.T) { assert.Equal(t, 1, len(issues)) assert.True(t, cmp.Equal(issues[0], NewUnloggedTableIssue("TABLE", "tbl_unlog", stmt))) } + +// currently, both FuncCallDetector and XmlExprDetector can detect XMLFunctionsIssue +// statement below has both XML functions and XML expressions. +// but we want to only return one XMLFunctionsIssue from parserIssueDetector.getDMLIssues +// and there is some workaround in place to avoid returning multiple issues in .genericIssues method +func TestSingleXMLIssueIsDetected(t *testing.T) { + stmt := ` + SELECT e.id, x.employee_xml + FROM employees e + JOIN ( + SELECT xmlelement(name "employee", xmlattributes(e.id AS "id"), e.name) AS employee_xml + FROM employees e + ) x ON x.employee_xml IS NOT NULL + WHERE xmlexists('//employee[name="John Doe"]' PASSING BY REF x.employee_xml);` + + parserIssueDetector := NewParserIssueDetector() + issues, err := parserIssueDetector.getDMLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, 1, len(issues)) +} From 3735d9ab70ee326bb116d5d5d268bc2e7a63f717 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 18 Dec 2024 17:31:25 +0530 Subject: [PATCH 062/105] [DB-14412] Skip collection of assessment metadata for unsupported query constructs if guardrails checks fail (#2096) * Added 'pgss_enabled' argument to yb-voyager-pg-gather-assessment-metadata script - argument is based on the guardrails check passed or not in assess migration cmd * Updated gather-assessment-metadata.tar.gz --- yb-voyager/cmd/assessMigrationCommand.go | 7 ++- .../data/gather-assessment-metadata.tar.gz | Bin 11086 -> 10503 bytes ...b-voyager-pg-gather-assessment-metadata.sh | 45 +++++++++--------- yb-voyager/src/srcdb/mysql.go | 4 +- yb-voyager/src/srcdb/oracle.go | 4 +- yb-voyager/src/srcdb/postgres.go | 10 ++-- yb-voyager/src/srcdb/srcdb.go | 2 +- yb-voyager/src/srcdb/yugabytedb.go | 4 +- 8 files changed, 40 insertions(+), 36 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 767c5bf074..aa76d2db7e 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -56,6 +56,7 @@ var ( intervalForCapturingIOPS int64 assessMigrationSupportedDBTypes = []string{POSTGRESQL, ORACLE} referenceOrTablePartitionPresent = false + pgssEnabledForAssessment = false ) var sourceConnectionFlags = []string{ @@ -350,7 +351,8 @@ func assessMigration() (err error) { // Check if source db has permissions to assess migration if source.RunGuardrailsChecks { checkIfSchemasHaveUsagePermissions() - missingPerms, err := source.DB().GetMissingAssessMigrationPermissions() + var missingPerms []string + missingPerms, pgssEnabledForAssessment, err = source.DB().GetMissingAssessMigrationPermissions() if err != nil { return fmt.Errorf("failed to get missing assess migration permissions: %w", err) } @@ -690,8 +692,9 @@ func gatherAssessmentMetadataFromPG() (err error) { if err != nil { return err } + return runGatherAssessmentMetadataScript(scriptPath, []string{fmt.Sprintf("PGPASSWORD=%s", source.Password)}, - source.DB().GetConnectionUriWithoutPassword(), source.Schema, assessmentMetadataDir, fmt.Sprintf("%d", intervalForCapturingIOPS)) + source.DB().GetConnectionUriWithoutPassword(), source.Schema, assessmentMetadataDir, fmt.Sprintf("%t", pgssEnabledForAssessment), fmt.Sprintf("%d", intervalForCapturingIOPS)) } func findGatherMetadataScriptPath(dbType string) (string, error) { diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz b/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz index ea97be02df8724ce7e2e6e1faea0b8f262c85904..6d9aa6a7dc8391db640664ce2a6c37fca31c2d45 100644 GIT binary patch literal 10503 zcmV+iDfreOiwFP!000001MEF(ciXm-`MUZQsG?L-uBn$_=QvSr6`4-7btG4kn`T`f zU5cV*ZYYu=NISkw{`<}hKoX=RKa$jG@0#bN7Ks5cnD-1oc5Z~{R-hT2TbwT}H`EqZ zXv~bzs6YBvR|Bry-EI8a+}&=df8*;>b8Ba_)!N-@G+K`u&F$UR<|DTK%`mv;ig;)Q zj6I5`ksC(Wz}?XPH@)WT`R{u?oCg;F;C!Vh+>Zadn~lvK;QQvIMyt`>+ujvQIb!3^%R`-o*215J#?-M`XH*#JhkAEhU~r)XT-mdY^rmny9K501 zJ#L;`3uEFM3#;bOCqUGY{$rs`@xt5J`cNmx+0>q~Hmit%nFn_`3WH84byI^|jG!6Z z;&1?LMnM3>Bmk9a^;x0tyXpB?D!QS^$}1$*v_zlTfHSU0dh zPnkh#;02&EEYT+(dI9`6w}iqo5QoSCeZGU)aBD92Ns&7vrkDv~N6cDaIdQ8u{a?!r zmDX-&q%-=37np^<=EYrm0>p=Y&)Hj0A4sFl1W@4S?YkJvnFOvQ1 zAbYFEzp>XM?zI3G1h(OTa?RL-~L&Ug4E5=>o?M|jwp~cCDvlYEf2aEzU3;#Xn^;fkcm$+}37IWF4 z4{Y|xuqOxeAK2(jsx$>Qq+;EM`*)SjaM*cw!o$FJ z=abNz7(rlMRrbnK;vJiZIPRx%^%R{K=-qg*KiFgEhC2hDK}Ca}q-wT%=2cA3i54y+ zN>b!k-8Ke-GD# zBmYY}a-`R~*>YYDd>KuhfRhuH=~Z`d+#gq-R+Ajd2*E>5qcBdg@P5hV!b3M&uotY6 zI7xV9^I72ezBQ{}i&@5fVpsjQ-e^2xl?;Lszh;7%2nGaFYa#*F5hsIvpT$eQ35z%( zgP116x!@^&mJ67&ffArV5!7D`O6j z{#;w%r^e4$%yE7Fx3#&myCv(tyUkX!`KW=v9_oMhay|Z`KDFKY)ZpiZ$LM%KH3NHo z9)fA8vUkzkz$|h9^Hn7u`V+aLGz5hlakL4UtD?hX2IcYyYo{oL#C zZ!pV7H3f3|0j>cMwUM0GOwev*Ss5hGJb@JVEz>@;O<0{fk3i?y+`F&>7qe5}3Klj; z#&SS$M&mg4!VW3Zju*e`NmU4E%mPV!!i7has^rOaqk~)k5L5 zet0w)>%WcL#lkU%RqU~n0?(eKllEK}!Y-?e*kR|nNEPTTNwnnpfm&T)%B&I0BdVCAOxT#1 zhlU%{C=0MuY>Iqx9PbjjSm?yidj%17@y^k^B%!eq;3Y;tUA@V|@RK^8{X8ha&=G!hr z*C5YA;GhGKEDn@lOkv#8R7bXNY21Q<-oP;l1_&;?aFYDXbK5-6FfO3DWQqk7Vl`6y zh82A@K*4~mWj~sNjrhb3upXam_X2q21y^OnbB##R04NUd8zuv2=7XCByewIOO*^u< zBUnYC$y8L2ZpLp+q&5Q^tSj||xEA845Z7snB5;_&z&>Mk$Y!3!$xK@xfi^^Jf6n;? zv<-~lEI-S>6{3}JEySHH5DvP$Cj`TVU?Kz?fye>Q3XHd1fB}lnHrNg80Z~4)=O9D? ziW(9l_UvOv4YM*Mx-W!f@np-FP@zgp-XZ9Sh36rn-xV{k)1w6P?tDwndcrrB&NJ5o_++GY!uGybOIO( zBqsIJsTD;n29t7O&#YNI&Mo3$16NvYZ{Z539MwG9S=J!Ug$9=Ky!KIsVPw;AD< z4Z1?;3B(#eVYVLK38LwNaIp zut%p) zAOQHOs4VBCRGA@9wTS(r{N$bXWT8Eou_v$hp1j$6GAdV3*-42;K4m32M3H@BAwNiL%ab1@T^5roRd>ZTD#U?N_1u!Jc_BcpF2J0TZah>#zpz=}g1+y$7(HtLM$ zIHH9gKPwP2(p2Oq0}lmV=$T^73H2l-G#t>xSrQS7tuSbFFm?v*jW89XMlR$UWd4Me zU;y+izn7o>US_AK?9->ryF44Mu2Lo6Ewu1GD#_MD(4#1j7^!k6MFtrtHwS4!i**d=gq zZiPZA=Pm|=Gzq5Ev2>8#oaWhcnQ0JVHdvm48BJtGeK;HpL1p5J z&I}t&^IkG@G3Ax8;wh6<&B6fLgD^e_<7AF<2uw~OSI26n3_vN0lHpjHayZv?BDBPi zv_Sp=njl=ID3Rv$LG_|dr2%N7c1!d{qC^B&(5m>Ee5b17NnLvfqjvdInW0`WEy$10 zoH`P>j4n zD~Yk{v>3r=TBop-_sJ7rApV(ou2oP?f;tAyNZ2n=r(*t(b|uEE8ViSNrnAd@3d=;< zSwM77BPW45F@U1F4F6Z5$+OBm!ZH!wBpqWVk2_YvLf3 z-~e*7#Z=~O^4MJ({-nOB*FPA5TJf6(ZI{;}DGj0@@$+k!qLzNZgrl0KJZ0Lyp>LW` zB@rOY+F%VNQpdnVt%Ss>*B(o)fY1o8q-dFeVhIo=wLW6GgKRDk3hQQ}4O|90g7z5e zUZ4{YSjs37fz^TOaTplpd$EDUC!PzOMUM)ix~iI^g`E#3E6VLHE)9OXyCv_r+}x7; zGi7y#@-r#LGA_)`USvOj8sgs@%<9|Dun+8iYfxEmFS81?dK68|Y+RO?vLz@=v_&bb zm}OP57nYTRYd+*e2Jt!;S;6!Az)%rpQsqV*X`x7`PTOP_ud#hfPN5wO9+y~56UH1y zi>!aL%DZ~Mx+DQ{`vk@?BT84}_T?tvnKTB=Z4^ERtbDQ;`GD=-X&HZG{SJRuL5QSZ zq1uASs6`y;p)kwfHr{&+u;n&a(2xvVH6!xy%HqI8v;)^dEi-xrv!$-}Lk$bqR%(7)j ztojzD#EKAtscw21n+z!FQp}CqjC;vI1e?UZS$_ zjZH~e1*|!v;6?ScALjbSYNnkqQPRM6QK;-%Wj%|hW7?h(Yv8bgwPAkbe_xVv(o?%Q!w3r6lQhVizlG-!MKoLTbJAAObu) zvnUn}?L}G}=Ns9+?}D92ip z(3n^PQGTc#ggacH#E z7zZ%x7@d?`vo&A!Zh6L-dKsuMC6f%o*OF-^hDZ|3OmN$iru=%Ue6>Of*O?1QRw?Ce zRZqEN-Ob(4mu`T`(+p6@J}2WQaxBc!!TAtX_|Znk&bcs zo|XR)b;Q!<=dygH9n{jt&CliK02c%Xa!*wL+~gEgXQoz)iN&Qh+7fJT-L{^Xn+&yz z4hn_mh6}?an@EvA4~!XYtj3@)36SdCo}XJmZC(A5{x05QSw{1vkX#e@Dk0yB!zOXq zmn+S;lU1w{R85Sfd9HJn+~uTHOOl(mD5-0j>uERxr9O1$UAmCO(oaDD!gj2=C7VRH zj(Hb8a6DtCUD$?}9KF^8&%;^pU|3t8C#{P=H4xZxaY-OjHIS=Q1R*2dIq}bfB&9nn z&m;drI;&&P$*n6pf@fG(!b20mwpn4F=J%5(jEc)`~g$~16A-WY=Vf+^7|895w zQ^XRX999q zAoWEM0g%225>B1`7&KL83;or|J{$?(=hSL|LocRb*J*z zDE`C2{Qs2ecJ^O+KWTDD6LtE;o#c&if)?5)vI^29 zo3&_`R24scV&z&H|BQ}bj>f}Y|5Zf*q`StgDuJDxmKna4G_)4pg@pwku^}>{T9?>T zB`u(B4lo-Q?!KIyUW?fANFQqZ`a!R+?_ZCff)ya4eCFerTS0PkQy>uGi9{B|XC01; zfEdU%=$r~Y!9nBxDPtAals-%xlTurfXiE;Et4b4=9JAYB_LY>rcS#30Ov9D$39kHP z$(e-*k?0VfIA}xXwgGq5;;Vw0e#T^lS-Yk2SH}ChW;u8N{ugaC-R1LNH-7(Pdoy4E z*=jW!5BC2)u7~*lFBbpbf?jEO9e?|S`8*11-1;yHBY%S1@V6j^&mqR1VfoH*Rb+4& z+R4Amr%1AxxP*rd4&fi_gUFv{AIvQ04v1#fCXo%i%ZWH)Ah6vl=XdSP2$p~NXJuJ* z^b23t6B@*6v6N@wqL-7bEGbz+%QYh#jx&)b8<8qv;I$cCj+kHfX_2qVyOQMX&&?6P zAYfP(C9M9;S$w+UKdkO~7yl2sCO*vat^NNl*8g^P8@oGOn>+OV-`4Jf|G$^(*5CgG z<}<p%WkQIB(yy}(y@<;G5u*(ndvg44k4W$#t5KQ2f;9$D&}En>SlK6DCU$rx{V zO>G60&DtMS%>gTw3j2EZup@}Mt0+ksrc}pVip?L<3zIM9o|6|rm z2bioO(^tNCp*Tu~O$lU%Gck8ujxTa0Ye=9d0f4o{==94NSwd+~B0(7;cWB_ksZZ$ok zgmEE~_*o%)hrdcCVOsQTiGZC2-r^SDLcJzNP+a5RD@M4+{{taR$NC2U8<1Q6{~uaS z@cx?!*W7J2S`YvKhx@p0=Kqp_L+Ix_5x@N$H^qND&7FMw*VulD|L*0&KuYhv9>>a$LTVbN@5I?Ph?+en4^7H|-!ARa< zV%pcC4+7AkO#IB~81HKEuRGY+Q;@+(fjH>&4v!%eEC_5?nQyh|71qS*#azjYt*g>- zzs93!QaX|m+K7@_3}KePRti0oxESmI|JUA?ekYM-@$dc>)igE`K}>>r4i0mIfRWQ| z&7z%&%8)>iGYQZXXxmBVZ=ZYb+pAttB*8eg^*$s;z3slG?)H{npZx#u-cyVb=c~n} zW9iGimWX8l5x!y!XO>xnR)X%K@e29dcAC?8Q%mChr_}{&0WKM5vS?G@;q2MR5#!z{U= z=9yNNufqzKX=fqIT|<;03`>z-u%+4Tx%{mBWzUU?q#RHxcjA}bgmftxxpA;-)n6-4 zqTG%@)DulXg&dJ1tkFz3#9-)R28Ty1)mcW&Ak7Y-Ms^1oi0GnCfaTjgN~EYh@=JA& zI$#c?bE;i=1r#?*l|#Ft(&V-QZA#?f$>fomG-?B)yu^te%{{QSL z`G0axa=E9uy_BE_!!Kd-@rCip^c?Rq#MKQSa z#}9;8{vB)pQm@ZDlVm`C6eT-d5yTOS^mJLpV zxjIIA^njEuK2o86CWFkkdTDM-@FO8BR*2Pd>{d_{e5QP{6Am|_zkSUF#9j} zl<+^F?LB$EzyFNw|9u|ef80xm`2X$rfAGNw!TpW1G5SFm)9LL!-eMO36D0urXKq*m zRfFx|4NGGf072gOiO%KB^EX0dI0nlWE7B7S+nofnT|qMKLzAqqROGMCQ5zTN|+Ug_JJ3IGcE-nVwRkggTEpj=G=+QUy2p;T?c<29hm})5f^f zbn?e)T?bm~8}kDXOyg51K73vxMfYD}Fbi4!^=NtZj|F-rFf7+)7M z6}xhkT^oT19Pv_rb~7T+T;^9EWBE{*%xDw}-!M7%c|tka0VOT%M!rDdcDyO<NCFdwFGIbVZH% zHg*CAWP+Z#q?l^KkjqSMgAh(QAR1odj(20<%&tU=K;ZNdyVG4`Sh^v4o^kezGgsOS zJsQiSRLEqxY1uVnZ5yNmbL&Qir5s(84BC|NPrezZJ`kpK z%NZZ6Y+D%E-*!#w@2_Nw(F#?ozqKazZ9=&Iufg)MH`~U4fA&1LYo7nuf3m;3AL;-1 z5xyDz`_Vn)zjNO4og=`1@1X5>!)#~#XC3Y>7Pc8~JKrMkZ86(}8iX^I6SxMi9n0jH zAV8vy=?dv2hr*3}$yvq&CRpH3ngv~B7Pv;A~QT$wj zp5j5&Pk65f17D&YEc3;PeZr8zU5#7$g8@Ca38WsOg~yzm(E7c=h4ZB& zg4vi3B6M&&I=GHIJs&Kfl0an;2Vgv!vddk#I4gG=4=7uovyP!V^?`U#iH(>z(!mieeeYJI8-|a zJ2-xbbgIYv1TisPTo+}APGP)WiYk8IIk@KVuEi%g1GLYWirT^w3rnvj@?S1Ue1$;z%QtNpLld2uWRope5>J59@d_RuJxW>9@}Wq^C(eJJENMDBg=|nt3@%vAOGpo2 zNZN@NVyPE_NpTVu2FH8vgOiNi)k!k@;n;b%eE;Z*{flRz;8>cn2b6xzK3GM$VS;S0 z1^O3shn5%?(WWf|HFAp$nGEkjKUhs0l1;kt_UJ~AbqB)977U==_tt!4{=(%ze}k~u z#@Gfln$(0dCq^H{L)V;E@S9rs(B*U9oG+{qeqve2Z&)9UzO9-%X;{#e<)&5E50ZsG8={RCgRMY{U6Xkgk@vy zSf)fUQ8}49|7HSFyxIHbFS}N){NK)JP%g&oem{0%YBZ!u&=D;kz?+eeTNQjymXMhJ&3&A%V$8V_?dl-IQ6N$TyJ78 zN#~x8xk?EJ{vr$iW~-&3Nl-)GxmjmO?aGN?8tuG>FK^^L*%TL1RCG3gh7Do%COwmj z<^_Y;OT{$yz?!B8m}V@3+0v2#dU$|;%uq36qbw*l2?Fl*ZmPSLjs8u&o7xRDS;*;N z3GaaP$q@`Qk+*f4W%Gbhhm8+ zlvow?(+cBFd;)n#O&_#fos8>sL>eWD$;Aef!5a|Ef+50n9|Cjm6hOQU7jfWIn$DKo zy+XGhLdw0?Axl{f;bHVKWYL|k*yCAia_+xrmB9ooRs;%c!i zyCXK)zHo*!XnJinn?uVNv(@+`yS}yaTdf80B0odEKj2K(CpL<#t`PW*!xYjed??Dj#?6$mAX($byfu-p zCEhKyMsqlk{1v2UI&0X*fk0^1$4(V$0uW2sWwR=~@be_ng`f#Y>rIG)Q}5K6u3Wc3 z@65~>vs36~B4r6oPw9b5QK7A=v@Vix9b7#5B9;@!Qu~zmor>}q>pSyNVAa6&zTwrY zs*_Y@N=kPs489Pf+=K}~O&a6GN$g%`Kg6@c>x922B)9Y`>pC%k)p`8ZK3!d5T`Qft zPZ#4Nn6<~x7S2`lr4>lJH^&uSU>LOLje}-GkN+ff+RP`BPG3^rOdsPJPiL$1AvKW} zqgT_R^v?6TMHM!`Jaw%fo_cWsn*5nqMFhQLn|RfFf4^?lE4;&FQs&^xd?t}yxr^-i zZo=r;RjPP0LWW|kR;l%A<7r34=6JEiZn57zYE}C6Vyjhc9E$ii z#dhg+vAvfEk$3uWut+z2Zy;|aznol*U|cWe=~}aKn8&LP=*ejp-{x^g&mdi@HD9ub zhI)L7Pb~f=D!Tj`awT`4rShv482|WVe^343X?pGE+iE#K!d#$mbo00ZJ{~RbxpG)(%irw^6a`w< zdAb9I)-b6Hr2_MT##I}|_EA58CLYVG_eXcDb+n4=Ev;{Vuh(#p@_$~Oi|k%{TD6Yv zoh*u76OBKee>EH0sPXMi@YDGVxQ{nStywD8Hps^&6LdrKt(ZVknJUv_4qGcmbA+FK zz!h~|&|!AhfUjJDaXAP3Qc4B9`o9MAjEo_e{N2}uSa!5L)mG(jOW7#HxXNSTg=hR@ z>~&kcu9h!QHF&J$RqB|Tf!{DBlXpzyd?mzU0C zrkB2+L8o3{U&Vb)>1rHv-bqI3%?H9l&n9KZ`dozZre#yiD=1%w@lnj9Gqj-O<%x?L za|vV&DzTW&%IQM#x5R#KqXZ8(3@QT}39UUlW&P@9m4P+G1nGX6kLv8oC04PWU;2y4 zUS5K1eO$z?Q;;(|wAUfxuj1_EDK#6fs)yF%M$@6UoVfq<09!Who%-TS>=J|@+ib&Z zv=Q^&(1n79WDFYy)4fyqt%qj?;E{v|{Ffa~&!u0BhxqnQwel7dwX27~cHl{D8^=`b zW~ox{wJ}|tC+r+GnvJ76*H3862X5G&pfYnwL>wQyzmB0PqEujfFYRwiPfKBm?Z&*>@f(Kx6pdDcT{nLvn!*UoFseZ@ zKWE3J6fN*h{YdMzs2IgnFi2%b%cxXx#*g}^7hrL4-ULZAe>j|{>#lAO$` zf5JGK%yicO2n=ns2s1_`L*)KAqbKrY`fvh+3=nS$r3UjDV4pBo%{DH&UDAZv%{iKL7{ zZ?q7hv&CdQ!Rw{8-7aV72f{6Ve9tXyrx0ga=Ck?G*dua##{=z{9svW1314K${_pVp z|2A;s?cV>iOZX4Q`QP00-Dv;+eT3-z@9ob220(`2=$!7FgSoR?9n6JXs&wzUTp4LU zGi%T5JC$pcq8*8y;knP-RA4!i8&t=-Be`qqr7W83LepZ_9rQIC zN^gCK7FhzA()9Lbr*>!SPVJhFv>n>@wrd-Qc4ziOyIx|#&g{;D&sCa}=kBL@@9D^H z$bsGP+c zhPg>cxGPZA6fqi%$emkg-kZgt-ql6z!3}>?ZFhUc8cf|7Ihz;983`|o{jwG02Zpwfu*r-q<2+w;6PT4q;&-am zX-V__Tu2{*R=qi-*OCO@XpTGQvPpk6#2iky$$?teAAiB>;F0b_xnbB zXt1I2=9mrJFOLis*|T62nPb;x?Llc^92xt=5`zmR;M$q2XE&w8e(w$4?s4nFo|z-h zoY}QtIs&3b^dB2#ikIHLHu?s&oQ$0bYq5#|%zU^*C=5EG)QwGUGiuG?7QsPdD-J^d zlQyVStItZM-z`75R?&@orY(?Ei(hK&ZD;tJ?GJusuZ{MB(TDB_{oeePpM7tB-1l`m z{U_S!jyJJC>fHI4&F`ZAH|hUIb7!0MzuDYsZf)(N{%=C_2mQZ?OV`JX?x8;ISU{>kBdY$f~_sAt4n3l~v*6RYBeKPDR zL+M@V$T%D_1hD`DQHKQ%b`P8y-9l#x2Enyqo|Pg)di^5htHQqsES2_vcm-r+9l5`b zeb=$|I+DFwCE(rOaIS-FCK?F593F%YI+H35fiE(#Zo&P#O1s~0zdPko=y=mn2)T=t-*Pc;|k_^4)mmF`{IZT{+Q( zR33U%3GEND6OxEA3w@@4&Ss#RPSR@TOhe*+702quQ4mK2-SWK%eKPjI`vqEWlBYa( zTzi3+9rmQU<@~{ell^;Mck%y1Bsu&AS9pgA;3ogS+1x1l|J{v^2mgN`*MlSfOFD9- z*Rs(HUJZO14V^Y8Cn(da{oYA;Sb17ab1X9g4>gXXB+0`2IhP9$y?Dl6utw@6;gQ29 zp&tbHq`DHbocqMC#&4a$aKI`#1SNjW6fqGD2&9%o0;(fUfqkFJOTGzD)T#tXKj~%Z*Hu**AF*+Vl&Cr=%L|_`K z>|H!HF-tsnxz0MCRVzKFMn{fidmQ{*48|y=ETRoY4F1SB>+DxMVpEQ55Xe>sMD-HIopo`QfzgiiW&?)Y&(>149R-={n>7d30rlHo*ZaDqW9Cv;;$#x!M58 zW_qj){-rZ0vzP5bXRuCC&=mGghU_gG-0rYr3@{nq?{yD4!%nXYcZX~Q9AWGn|1 zCj`fJW==$zCZC^LtyJQ6#B@j`2mAfb@o;p|>9^J@%$hJrOUv+U;)LW%==Kko?d#E? zchcWC_Vl&J=Q697*^e(+ePUnMz1Vfxk3X{DY65;gf!J-oF#seMh6&JGqgpDxHja)* zL*ut$t6VzaFpE95rnG0z(MkKB2VoagNXU1+V9yno1$UW=8zm-g1o*xvmG*ttg}$5} z3oAMJ<%p&AYb<;N0LLOKB^f!U_2G;tRV#J<2v8aVrTh?SCS}$Pr!iH`Q6?Nr%p=o_ z2+9mB6&oX;T-U!sE|%I!>%EeQI*JoYA|eIJtPPWGKxlDjf!tm$lA61;%qf5#f!-4D ze!76C_+t+du207JTU!l@86J;AsrLycoCOeN*`a&QKE!?`l*+us(2QT$z(3=R8+5o{t7fzCYb>UbSImU$)myEGsLaauL-!P+( zCMX!NRSV)V*oaTu1ncq1@h*W!et4}Ro=ZfE0H8R)Zx{?(vjE&I;AP7KY}Sz_9lBbz@8u^)%vA1WZl z+=gK$yjo+G1N+>J-H0_e8r7B1N~MHr(CqltaeFX$+v^{+o(DkhE6~jsdAl*M06)3D zWxA*Vd)r%^8}-?BVh6OmAi!-Md0LPBddAFpG8uTi(*O}Uc5SpFwp0%voKN6lhJfUu zUl+a$CW8x~Xj)MWlmu#mCNLg5Qft7!g6;P%ozVAioLZ?|0(-rGVO#H0Td(V&RRQw@ z)TR=!`xI<&v&>p8Ru=Ed?Ccpf^@xhLb>Tyc)VP+#(~lq}jncW34FI4_5U|v5W_Atr z3Y7fPnb?zL#K&c(1G6?4jFs5##-Fs<>~x{bo+&eEmLRLj5Sm78>w6<4NIC!yI*|g6 zol^ZNEd(!iNuaF<2tdZgmPBfDnFz3m%&oO%NuhIVZKJe=(5%|5_hbNhKRaC`$Y*R#f+(|3kiJ6~6NsD4q0E#}uk;z158RGMK!MTM zLNy&c)DeMZGQj6D%4C2ov@5)mb1I5N64LkBYfF8#DMn+OXBI_F^B&`e?* z5ed3j92C&Vc-tuKkmbxoFpWB5CqV?(*GKXlb;cbM@hnK5l?WNB4+YA=LqQjMrWj>H zJq@-@7sPFn##&;L3EsIFm4b-InB>q)F0>xh;gqcb0Q3;QYfpdI*x4ET^eOka{PcHF z3RNo<;zJ|J3>U=xG4nA;xpFXNG$)f}bTKAS(vMO0Q=|*fGz6y!c%TWyGi(fo5AcU5 zF%CVEw26f;K}(1f{z-&iOP*}J__4WA8vn_zfP+&z5+-ZvVVFgOV6vJ>2N@SmV;pym zjlr-zJQlR1yMwe^`eSCbpb5esh~i^GA5>S-R9*m0)NZN1 zNR)`+3R;yQmq}DrJgIBvaM02|X$>0uE*d;a`6DnW)$SwTijZgh@9-#RxzK?Ko2 zmeUjxb7U3b=}%{$%g;bpQ*sL>O$HIMoPfMQD~Yk{uu&M>xwm7O%KP*QFcAMte9tba zCP9UPBNF!W!>O46qg6@pswTprn(6#BpTaazc4puLQ|3NqeVhC4rA&~<@EeQ!lsNfI zeLZzu8ZA_;mCjyN9|Xb@hXL44$Q@A>mN$PI0okI#Hjr+e4~wcmw&33LLYCm9N1V-+wxjzZIVFP3Kb$oGJ= z=utsb7u8AhsQtlWWw~&~rNK|ujpRL-%SLkLLsLi4o=GW|QFCGRGW!A45dYp_R^PV! zU10xPlL~Hoxmlpqqma?Vm}8O@Zt}$)O0z|gshnq3xf5whDYF=mBZGJy%a`DJ17N5K z!l@=AHj7ZCQ>O(eo7dO@C7JLZO9@w4rV_>+pvCrovdX)9w>l>Q$@>w&uwqI(llSwl zfM?Pe%)g`XF=X25UK{|nduJN{#>yD}u7VIrze1G-pV1q!RffVWlRJ3t&%l;DTtP!J zaM6g!!)u!Z6VVQ=Y&Fm51(E zSG1%*w_#Y5+N6?cz7@+*z-aaoLJ+FUV=ag#xrejLOZa$h$u%Wi zO1M!NaW4f#ZPS=M%Oz5k6gAE?sYn--=d<668vKJCE@~x7n2UXi{nSXLDll6tqwNLN zM_CvZrDQXVq)}0U%F!#lT2^8isz3sh*0ME$`cv9al#66ULy{}CzK>Yri!!4w)na?o zlD1=653n4VfoZ(1*I9k4We}EGWl02C^heUI;Qo(#wI%RJ*%p1`|acSHgFXfW&@w}650OcY{CvuGzu%kjW2nQnOw zr@ZkZ!$<1~)LfKsGuc2LROT=bO-YDQl2^8!U9uw0aF@W`LtP?+FH0-&A7GgZ(I?Se zS&kzACfJSymb6>-oOqoJZj2 zM$#)Iv4pQyp3b2nYH*8ZHBXlBC~D`9cY8sj#(xblBUDJp)b7uo<<)%tP|5;DPx7|D zaop<@bidwge_&-cpXI{C*8k$00v*`>6GAKEpe3}CnFnNE zv@n{lk<8Dj`Zb95r>&ESBZ_2QV4-enhr|e9&F~bg#?$W=3w;0S+uXDYb*5wq2fef` zgtJSr$0!MtsW>j5VN5Su>AHLdVV`;Ro@sxGqUze_XH7oR4{O=u=4Wj_+6RGwL>xuE zFgRrxxuKQHYI*J*t#dZFZd+U}42J4Q`;npx(*rQ+vQZpNLvuo_#Ys~b1W0w^OfT%P zwyYRSe`oKpMx_T*NLC~PO8mQEPfHR57uG3S=`5BAsxDyZzUN-2cLgccQ{&VU>kGQ> z`8tk3ri-XNG_;7v0>vsEpMZ_CPMbc*`1r@hF0(8UqpJtQx|L*MW zZtXTVH}L%b_GaVZ{QtdNcR2s!Og@f09;l~)NG@a~wVpIm4+4n>**TzS>=t3rP&pXT zNQEI9;9c&Bg*X@C3?t+Ps|Bn9=#CY)3` z6GAQTFUf%YmB}Xk6C&l$b`;5f_oEjCCOP?$KijeFRK^m;e>j-&?{eMF{wwFxMt3y+ zk8ZaA+l}4A`Oodf_RfR-zmMzT{O4cn{O7kiM)>gEh2>`t1^IO(|ChPA()U*5FTTFy z{6D@WV%_pHKsVa|X0vGjceWnl|9iP~oe2@hiaNql;{ zf(x-~YP7$^k0=AnC+hI2J1H9D6fLw(WEG@IwrcStE%$!<#I%}*e+DNn2g81+`>G-u zWP2Mns?_ZCOk;Q|t#8l#OB*Y1V%cs)MM$ymPg+2WabPxVTq8U^TZ!1o!077-#$l&x z9IVDq!3vO2KJ#+I?J(V7ED(rs-z-bo=N*oUfB@u~w3~{aAkd_L##qHQWe-!wq!j%m z+LA-)safdd{r>B&)6(5 zYqvE13cSB-mUH*-ztLjoT|WPI9_;^pTo3X8Uo8H=1--KHI{Ees z3wRvWxcy-i#lZ+Si`;?~0f!iSg5^8YQ;|V1wBO_|pQ6cQ;t?JWID~(w58_~we=xD# zJ0O~yn?yGBuSQ}YjKFqJEAB1Puvzipx5~0;O&w2#3Ju~kS<16;((_4Ho|G)5yG69TcnBryflL=5cAJ{TISE>U1{?6ZF9sg2pASc35!2-m+S%hH>-Qz#s7n@i4XI9 zZU4W!OY1+4-JPw?ogMQ3yAS^VUanhz{}q_e1oKya{I#NPIwpI82gT*WPMX;%57B}X zVD_@}s?!~oq#om`=j{BI*awhojfSvfh&Q~Zz6F)d+aFZT9$Q;09T@vZZE47IJphZ- zGq%@aMMIW+a#|k!sA$E}w5qUf&lb-2nKIwdw)}9y`o?~*fAFmI(s(5;Fif%&;etM- zb(i?17Sp;=PI?s%pHY{?9?*XOb#6Q`jAV>+Gdt?_j#JmVaG)sJDz6?$QnrA|;qsG6 zJ_Vax$$^>0>vijkc71uP7yk;^ey@FG4EBvm_vGlPb6DXJIk=F$Vlc4I^d@kjK_9iX zdE{X1H5sJ>m4la_=xg` zT@fQFuJP{`BdqZMKnTmVzrz0p=9d5em(2e+cQ=~NhyVY_eOx#5e@?(5^b6dW-+qjn z;=i5dPBH##Y(K<*_i|w%W$eH1vEJb!MOBA}aqzOe|8x2PEm)`f3cMiQzG?S=J~<}a z27d9tIO@FV42?cpXTO~Ezzrqcj@z${LFc~>)}Y&Azw>5r-0K}BaG=i{{FYtP{#T>l zIebUA!bCd}e%=XBwbJJt=mU1WfxN-Qv}-^gY(TqZ@iVhSysN>#{oa9*HR%nM7KiQ5 z(FuftC4tQ%^Q{)WqMF!NS}1w3WmOs;)Ob8jOGh$78&EQfAOYN3Y`7i0Z@MD>4o zUI3GOH#bTik7%y;JWPt>0-IS^PVHg-S?) z6pT%rtG;Y6|LCa|LSWfkMmb&`U1+^GvBK)L~`Av@;OptszPfhNZ|=u%+JY zx%{lcWzV&V6mKz7V+4y_iO4mB6=W3!EsE{@-gtgjfix>>K53zW} z(!X(w8Kl_()X4520}+>K(_s1bPSRPbkJhESCtWayadE0se-9M5s`Yd>0rnoSpLZuJ z{i>?{mNOw_ddQsq0CjL%`ES@H+gcCUwEzG56|Mj5@9yuv+Gpo~_xI!TUr!SLy7Hfv znDEwvZwE3n8-59sk8g}Sv#B$)X*b8y^ak_RaYC$$-19gEEm1H7UeQ3-Xdz0x$&?Nq zn;_P#ZSqSoriv4>na^zO5Mvk8Zg>`uWy7;*u8xrb{oq@Q4XzV(>Y*-F#K+bmWJAKr zPr-zbBR;aKd>{ew_~*1^5yFyo&S9kam6eRotYr{BL3BE!;H zw%Cz&3~bklX1ju9I)^4%VYS%@Uf4liSZg0u8ZCT!-)~jX!+ri87>_l}k7z(PL{dA( zAFX!hsM0KS>K_exl*p`+49V1y4AUkD^L{YIJC&iOf9e$M3qO&*ufX;OjC~=ciz?2X zpfpocDwI&?QO8ji^g*iNAwTp~NtaXE);JxETg{x`R<`vo``RaMLsu~Q2GVKp+&p$h zkqL4OKQ72+$*D23+9Xcs-YU5u!TYV=H5gx)G8OZF%bc6w%~J$*VE#LK<{I~`jpajK zGNV=~e8W6>&;EUxvrquzxk>gi?fEHjKW9RRLdd@vq!uN9d%^oZ67FmNF<_qiEdg(k z|F1AkxBvEE731^YPZDDJ{}}e4gsU=|e(GsR?WkWEKkVV=pYix5hBR$Hrj5r9F-Wwu zAg-B^Z-03^@o}s=EbXw33@Wwe!mH^07U(cmrP+M&#PC=3!%Mtt=L~$yogN_! zJ?-pK7tRCg<{OvI=kY9(%T2~ToNqC&?z?5RY2J1F@BXN%XWUZRS+|$6ZinPAJ^@4X zA{V`bxprO(vt#}24BEoxDb{G!>c2eJtbz*Vta8k=N}58NcM7>ZNA4-)XlqSik^T## znXQ)=HnwF6e-?8)&|VQ9!_r=BKYv@>7!!p)8*TJ*S^?jB5qMksPsMt_?*X6<_TTN zBX@zm-wgS?d}(6u5=jMp2(|I$Yi-`+0$c92c#o& zt44;Q995HyI+C=p3SFPvP|G@9Hu6xEcAZb}%K%YPdWW||gGNK5jiL~Kno_}w6sOqD zI%bqU>#fsg10564Xn7LT=^?XPx2^W;nYG^#HJkL`KyQV1;px((c`KVwp&90XAWZ3&Gd^0`HZZXN+dZxSeZ zZHduyc8#unAOqb1n+~tgE*VZ1HbV#nD$pGP2rtmW-Sg$uaA8w9V+XI8ffZHor+A2a%YB!rmNLcA|G9h4=RqX8_8UI-ax{QUb0bNcc zxC?Vn91##$-5T`!Y;5vxo@lvCAgM6c zw_~O8f-LssL8ew`Lx8$j*~(OEwNAa;#mD1LyVtIQ2rXN^>Tx#nq22AlGa{j5)@%mR zOHfq@0-O~ zkn~a|X*u%-L*QNBrd)N7^N#G#N9LU`(AE1?Li6H9)Gz4QgF%GR36}Z7W}h%*@K(0B zgI?d&i{Zh5GnXRx#N~4wdC1j;gLluBZ-8OXpV`7UJWbw6+)P=%@3EdxI2kN5eI*9B z^H+!*cxE#%15p0@p6XE-F-Qx_Ib7mRS_V64!p(Nh5iv4=v<=E0M_7jH+MPQ<9@Apyj(9)m~y!4`C>!c=een&Oj3u{d8%) zG}I*o#_l7C2@PW1o+_13p15-Cs;CkX$ykH7DJCLOZC`#Zls&DSmMkC9J{op5#eJb&S1P z2x@*tY(z&xM`5KYVyO9$YLxKjl|W;K3iuKoS7Qy-(f$V~4ya0+hBBOMg)T{Zt*%81 z6xU|e!4uTuP`x;Kf#ZjGylTvM5R+8JbvgMWJfP34IsPzF*4P(0_o?ELpeD_)*@LiD0yYSG(N@T065O9g__?V- z$wXaJLR+OzGg$;j*0igekgkQFX~sbX<*nsdKbEc5dwBTahpZtjABs%MvHoKfNYi;y zE<~lo;DN=wg!J%*lnSC3X6enSMcv}1von+#z&N7A1r`E5_Pe0tUf2jf#oXk=7 zh|=%a2kTR=nIMnX0`&`eKua`>XxkQn8u^F~amEj!AFQS=$tKX16ue3c%>e5WoR^xE!oP2|Uero_OD#>| zUj;8u<-&dtkeNEh5tiS_|(@SrC~uX%PnEeG6z>tQaZs> z4@7E%s-W8)Qjz8fK+OlW(8nfMM3D|i&Vje{3oxi}3#XqCKD{`&Exf>RaLYG^L}sl} z-bBJ2#q%G~K*Y;q%dt#}V4~8QS^s1LQM@Vs?fX5WR{l@x3Y3cp^Y6!QOxa_q1Q()3 zXA{RrQe0j!q0)#(EX|RX5mA9$Ta`Z|8#|d3|7i*iM zZXkethF#6LrdVYaye1X7j(%%R7WUlAyT?}g^Xccp!KWV#=&p5%?kY-#nJXT-0m`J% zzkSXt77e8!@-M{_Q7EzMxK1mKH}MISEPwi-?P}${;6kKUl9*gh15+T^W8z}CMqkLOQD{Z zS^-ME2>B#e0Q`jtxm?JW!{v&_dg42dxu}NrDrBlixU79bt zb9*16`GPjEw3NTuR(K%}*}k>LE;POAx^rmx!d*=+nETf5Pt9Y97x@|T1!B^IUhS)9 z9QB)jERruKIW%dKMdcmc7(C56vL zxp#P&G8rXn!eGQYxVZ!(rq(D6P9%Q?>G|#&ws9b!v(>Tl3)PXKB_6h3m09=&5~)H^ zM+y!mM8TEzC)Z*-aU!)(>F-pO z&sg7ihytSquJ#SD-mf}CRc57hr^4V1(aKHhxZtEQI#zn`rtmpc7~iGCJt4VeP+3=r ziL5TLZXMFqeGqbA=N^z_TtpcUtg~(AD(cn>BrVQyM;90doq6ME=FsEcNu4(TMWj== z)VI^ec*c>tnvSW7v>Cmcjpgb*uUk}MT9>D;4daF{E zy_4hm;HYwZ+-Mz&_>YxN^+TmoEP==yT?`kwR_G1nt(TV0r48eHInOoQt-}(!HlQb~ zQ~9}sCwhjtYP0>0MYR0KclgBO-=U(*pCMOf?{&7cT7mJ8KVBC7A3ROJ)Bd?pE4lbm zKD}?YE4>n5g`a~i>EJn(T&-R$jp4`TaEi~3qe|2N-R{%r+z!0&LXU=XH(I?C?toA1 z1wPjg>mB*KQ-`8JtD5GzP-qjAdQd7bA81^oRq31zB52~Vtom?tuW^J{(b2Kex4#@U zaga*CUrt5#AU(&;Zs?sXin)o#pVmLxEoIdBaVPj`{TsMXFh{dptu(jD$0ieWL;Ghj zfuu53qs<()SB&NezxjYW>bRl9?5-i-xd7vG4)&##3h4U33GrJ!G%t}ySq(=n9}VeY5gi0WiTHI3pJaR9qV%u#+#DOXI@eH zx{Qxv9+ja*C9gRiYRpxTF{s31HY2CokpEbm=hjLHaKorFppnqp^AYRUAghe5874^Y zdTt*vmrJZ-GrtTMQMkDQ*}7A~qf?NxFgDjA;%`&z#0sm!3({uIjl|y{{u~GjS z6LlJgz;@tC92>_}op!Ze>vu5S5l`4XX|-D?M_fOlBOiER+d*aKhKM*l_;i;WwsA+6}hrF)r=8=1aRlVm95` zb))Z9>XYZ;w|SrF%I<~>yPL1;Dz`~-u3N_2yXHA2JXsq0InojKmTs`uJHdWHk6>#y z-d~2cfW)a;XcleP6=cKQrz5--sA@hj3XI5uTd4cZ;!y9tqxR^Ae{6Jm{Yn$2ZnJ+B za)%YsaOUjlUTy#4EbyWLcV~ntPU>AfiAt~Wmag7G=&TiVQnudPvFR%Kh%Lc%nFCn4 zj9`zgp1+e9=-K%uw`B9J93h%zr->0@8#32rGh(jFW5QjL#Ar5hf ULmc7|hp!I*1<6WOsQ~Z*0HjUelK=n! diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh index bd82896db2..01e1b6b55e 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/yb-voyager-pg-gather-assessment-metadata.sh @@ -35,12 +35,13 @@ Arguments: assessment_metadata_dir The directory path where the assessment metadata will be stored. This script will attempt to create the directory if it does not exist. - iops_capture_interval This argument is used to configure the interval for measuring the IOPS - metadata on source (in seconds). (Default 120) - + pgss_enabled Determine whether the pg_stat_statements extension is correctly installed, + configured, and enabled on the source database. + + iops_capture_interval Configure the interval for measuring the IOPS metadata on source (in seconds). (Default 120) Example: - PGPASSWORD= $SCRIPT_NAME 'postgresql://user@localhost:5432/mydatabase' 'public|sales' '/path/to/assessment/metadata' '60' + PGPASSWORD= $SCRIPT_NAME 'postgresql://user@localhost:5432/mydatabase' 'public|sales' '/path/to/assessment/metadata' 'true' '60' Please ensure to replace the placeholders with actual values suited to your environment. " @@ -52,22 +53,14 @@ if [ "$1" == "--help" ]; then fi # Check if all required arguments are provided -if [ "$#" -lt 3 ]; then - echo "Usage: $0 [iops_capture_interval]" +if [ "$#" -lt 4 ]; then + echo "Usage: $0 [iops_capture_interval]" exit 1 -elif [ "$#" -gt 4 ]; then - echo "Usage: $0 [iops_capture_interval]" +elif [ "$#" -gt 5 ]; then + echo "Usage: $0 [iops_capture_interval]" exit 1 fi -# Set default iops interval -iops_capture_interval=120 -# Override default sleep interval if a fourth argument is provided -if [ "$#" -eq 4 ]; then - iops_capture_interval=$4 - echo "sleep interval for calculating iops: $iops_capture_interval seconds" -fi - pg_connection_string=$1 schema_list=$2 assessment_metadata_dir=$3 @@ -77,6 +70,16 @@ if [ ! -d "$assessment_metadata_dir" ]; then exit 1 fi +pgss_enabled=$4 +iops_capture_interval=120 # default sleep for calculating iops +# Override default sleep interval if a fifth argument is provided +if [ "$#" -eq 5 ]; then + iops_capture_interval=$5 + echo "sleep interval for calculating iops: $iops_capture_interval seconds" +fi + + + LOG_FILE=$assessment_metadata_dir/yb-voyager-assessment.log log() { local level="$1" @@ -209,13 +212,9 @@ main() { continue fi - if [[ -z "$pgss_ext_schema" ]]; then - print_and_log "WARN" "Skipping $script_action: pg_stat_statements extension schema is not found or not accessible." - continue - fi - - if [[ ! " ${schema_array[*]} " =~ " $pgss_ext_schema " ]]; then - print_and_log "WARN" "Skipping $script_action: pg_stat_statements extension schema '$pgss_ext_schema' is not in the expected schema list (${schema_array[*]})." + log "INFO" "argument pgss_enabled=$pgss_enabled" + if [[ "$pgss_enabled" == "false" ]]; then + print_and_log "WARN" "Skipping $script_action: argument pgss_enabled is set as false" continue fi diff --git a/yb-voyager/src/srcdb/mysql.go b/yb-voyager/src/srcdb/mysql.go index 3cbec4f9a6..97ab62e47c 100644 --- a/yb-voyager/src/srcdb/mysql.go +++ b/yb-voyager/src/srcdb/mysql.go @@ -542,8 +542,8 @@ func (ms *MySQL) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCo return false, 0, 0, nil } -func (ms *MySQL) GetMissingAssessMigrationPermissions() ([]string, error) { - return nil, nil +func (ms *MySQL) GetMissingAssessMigrationPermissions() ([]string, bool, error) { + return nil, false, nil } func (ms *MySQL) GetSchemasMissingUsagePermissions() ([]string, error) { diff --git a/yb-voyager/src/srcdb/oracle.go b/yb-voyager/src/srcdb/oracle.go index 971b8993ed..a1fbd634fc 100644 --- a/yb-voyager/src/srcdb/oracle.go +++ b/yb-voyager/src/srcdb/oracle.go @@ -721,8 +721,8 @@ func (ora *Oracle) GetMissingExportDataPermissions(exportType string, finalTable return nil, nil } -func (ora *Oracle) GetMissingAssessMigrationPermissions() ([]string, error) { - return nil, nil +func (ora *Oracle) GetMissingAssessMigrationPermissions() ([]string, bool, error) { + return nil, false, nil } func (ora *Oracle) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) { diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 9f0c328e30..1cd8ca8634 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -1136,13 +1136,13 @@ func (pg *PostgreSQL) GetMissingExportDataPermissions(exportType string, finalTa return combinedResult, nil } -func (pg *PostgreSQL) GetMissingAssessMigrationPermissions() ([]string, error) { +func (pg *PostgreSQL) GetMissingAssessMigrationPermissions() ([]string, bool, error) { var combinedResult []string // Check if tables have SELECT permission missingTables, err := pg.listTablesMissingSelectPermission("") if err != nil { - return nil, fmt.Errorf("error checking table select permissions: %w", err) + return nil, false, fmt.Errorf("error checking table select permissions: %w", err) } if len(missingTables) > 0 { combinedResult = append(combinedResult, fmt.Sprintf("\n%s[%s]", color.RedString("Missing SELECT permission for user %s on Tables: ", pg.source.User), strings.Join(missingTables, ", "))) @@ -1150,13 +1150,15 @@ func (pg *PostgreSQL) GetMissingAssessMigrationPermissions() ([]string, error) { result, err := pg.checkPgStatStatementsSetup() if err != nil { - return nil, fmt.Errorf("error checking pg_stat_statement extension installed with read permissions: %w", err) + return nil, false, fmt.Errorf("error checking pg_stat_statement extension installed with read permissions: %w", err) } + pgssEnabled := true if result != "" { + pgssEnabled = false combinedResult = append(combinedResult, result) } - return combinedResult, nil + return combinedResult, pgssEnabled, nil } const ( diff --git a/yb-voyager/src/srcdb/srcdb.go b/yb-voyager/src/srcdb/srcdb.go index 221fcac668..1c375128b8 100644 --- a/yb-voyager/src/srcdb/srcdb.go +++ b/yb-voyager/src/srcdb/srcdb.go @@ -58,7 +58,7 @@ type SourceDB interface { CheckSourceDBVersion(exportType string) error GetMissingExportSchemaPermissions(queryTableList string) ([]string, error) GetMissingExportDataPermissions(exportType string, finalTableList []sqlname.NameTuple) ([]string, error) - GetMissingAssessMigrationPermissions() ([]string, error) + GetMissingAssessMigrationPermissions() ([]string, bool, error) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) GetSchemasMissingUsagePermissions() ([]string, error) } diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index 025d5918ab..8b654a21a0 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -1056,8 +1056,8 @@ func (yb *YugabyteDB) GetMissingExportDataPermissions(exportType string, finalTa return nil, nil } -func (yb *YugabyteDB) GetMissingAssessMigrationPermissions() ([]string, error) { - return nil, nil +func (yb *YugabyteDB) GetMissingAssessMigrationPermissions() ([]string, bool, error) { + return nil, false, nil } func (yb *YugabyteDB) CheckIfReplicationSlotsAreAvailable() (isAvailable bool, usedCount int, maxCount int, err error) { From 1f69c1cc258b7dc6df2bc508c1641ee1882c9c21 Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Wed, 18 Dec 2024 19:19:59 +0530 Subject: [PATCH 063/105] Enhanced the pg_read_all_stats missing grant message (#2063) --- yb-voyager/src/srcdb/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 1cd8ca8634..5c6b62da27 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -1210,7 +1210,7 @@ func (pg *PostgreSQL) checkPgStatStatementsSetup() (string, error) { } if !hasReadAllStats { - return "User doesn't have permissions to read pg_stat_statements view, required for detecting Unsupported Query Constructs", nil + return "\n" + color.RedString("Missing Permission:") + " User doesn't have the `pg_read_all_stats` grant, required for detecting Unsupported Query Constructs", nil } // To access "shared_preload_libraries" must be superuser or a member of pg_read_all_settings From 736fcee98904c344c66d47d413db4f37b5d504eb Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 18 Dec 2024 19:25:22 +0530 Subject: [PATCH 064/105] Renamed count headings in schema summary of html/txt report and fixed all cases to add to invalid count (#2073) 1.Changes in the Analyze schema HTML/TXT reports and Assess-migration HTML report for Total Count -> Total Objects Valid Count -> Object Without Issues Invalid Count -> Object With Issues 2. Fixed all the reporting issue cases to add to the invalid count based on object names 3. Fixed all the test cases expected files for this --- .../dummy-export-dir/schema/views/view.sql | 2 +- .../tests/analyze-schema/expected_issues.json | 10 -- migtests/tests/analyze-schema/summary.json | 15 ++- migtests/tests/analyze-schema/validate | 32 ++--- .../expectedAssessmentReport.json | 2 +- .../expected_schema_analysis_report.json | 3 +- .../expectedAssessmentReport.json | 2 +- .../expectedAssessmentReport.json | 2 +- .../expected_schema_analysis_report.json | 2 +- yb-voyager/cmd/analyzeSchema.go | 69 ++++++++++- .../migration_assessment_report.template | 2 +- .../cmd/templates/schema_analysis_report.html | 2 +- .../cmd/templates/schema_analysis_report.txt | 12 +- yb-voyager/src/query/queryissue/constants.go | 1 + .../src/query/queryissue/detectors_ddl.go | 20 +-- yb-voyager/src/query/queryissue/issues_ddl.go | 116 +++++++----------- .../queryissue/parser_issue_detector_test.go | 8 +- yb-voyager/src/utils/commonVariables.go | 1 + 18 files changed, 168 insertions(+), 133 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index d018df373f..435adb925f 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -16,7 +16,7 @@ CREATE OR REPLACE view test AS ( FROM test1 where t = '1DAY' group by x ); - +CREATE VIEW view_name AS SELECT * from test_arr_enum; --Unsupported PG Syntax --For this case we will have two issues reported one by regex and other by Unsupported PG syntax with error msg ALTER VIEW view_name TO select * from test; diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 6b44e2beb8..899d852562 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -371,16 +371,6 @@ "GH": "https://github.com/YugaByte/yugabyte-db/issues/1131", "MinimumVersionsFixedIn": null }, - { - "IssueType": "unsupported_features", - "ObjectType": "VIEW", - "ObjectName": "", - "Reason": "Unsupported PG syntax - 'syntax error at or near \"TO\"'", - "SqlStatement": "ALTER VIEW view_name TO select * from test;", - "Suggestion": "Fix the schema as per PG syntax", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1625", - "MinimumVersionsFixedIn": null - }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index e047b08fd4..fa71b0969b 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -8,6 +8,7 @@ { "ObjectType": "SCHEMA", "TotalCount": 1, + "InvalidCount": 1, "ObjectNames": "hollywood" }, { @@ -20,17 +21,18 @@ { "ObjectType": "TYPE", "TotalCount": 4, + "InvalidCount":1, "ObjectNames": "address_type, non_public.address_type1, non_public.enum_test, enum_test" }, { "ObjectType": "TABLE", "TotalCount": 50, - "InvalidCount": 33, + "InvalidCount": 42, "ObjectNames": "public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, { "ObjectType": "INDEX", "TotalCount": 43, - "InvalidCount": 38, + "InvalidCount": 39, "ObjectNames": "idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." }, @@ -48,9 +50,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 4, - "InvalidCount": 4, - "ObjectNames": "v1, v2, test, public.orders_view" + "TotalCount": 5, + "InvalidCount": 5, + "ObjectNames": "v1, v2, test, public.orders_view, view_name" }, { "ObjectType": "TRIGGER", @@ -67,7 +69,7 @@ { "ObjectType": "CONVERSION", "TotalCount": 1, - "InvalidCount": 1, + "InvalidCount": 0, "ObjectNames": "myconv" }, { @@ -85,6 +87,7 @@ { "ObjectType": "OPERATOR", "TotalCount": 1, + "InvalidCount": 0, "ObjectNames": "\u003c%" } ] diff --git a/migtests/tests/analyze-schema/validate b/migtests/tests/analyze-schema/validate index daf6faa023..64ee24da7d 100755 --- a/migtests/tests/analyze-schema/validate +++ b/migtests/tests/analyze-schema/validate @@ -45,25 +45,27 @@ def validate_report_summary(report, expected_summary): validate_database_objects_summary(report, expected_summary) def validate_database_objects_summary(report, expected_summary): - key = "DatabaseObjects" - expected_objects = expected_summary.get(key, []) - reported_objects = report['Summary'].get(key, []) + key = "DatabaseObjects" + expected_objects = expected_summary.get(key, []) + reported_objects = report['Summary'].get(key, []) - assert len(expected_objects) == len(reported_objects), "Number of database objects does not match" + assert len(expected_objects) == len(reported_objects), "Number of database objects does not match" - for expected_obj, reported_obj in zip(expected_objects, reported_objects): - print(f"validating database object: {expected_obj['ObjectType']}") - print(f"expected summary field for {key}: {expected_obj}") - print(f"reported summary field for {key}: {reported_obj}") - assert expected_obj["ObjectType"] == reported_obj["ObjectType"], f"Object type mismatch for {expected_obj['ObjectType']}" - assert expected_obj["TotalCount"] == reported_obj["TotalCount"], f"Total count mismatch for {expected_obj['ObjectType']}" - if "Details" in expected_obj and "Details" in reported_obj: - assert expected_obj["Details"] == reported_obj["Details"], f"Details mismatch for {expected_obj['ObjectType']}" + for expected_obj, reported_obj in zip(expected_objects, reported_objects): + print(f"validating database object: {expected_obj['ObjectType']}") + print(f"expected summary field for {key}: {expected_obj}") + print(f"reported summary field for {key}: {reported_obj}") + assert expected_obj["InvalidCount"] == reported_obj["InvalidCount"], f"Invalid count mismatch for {expected_obj['ObjectType']}" + assert expected_obj["ObjectType"] == reported_obj["ObjectType"], f"Object type mismatch for {expected_obj['ObjectType']}" + assert expected_obj["TotalCount"] == reported_obj["TotalCount"], f"Total count mismatch for {expected_obj['ObjectType']}" - expected_names = sorted(expected_obj.get("ObjectNames", "").split(", ")) - reported_names = sorted(reported_obj.get("ObjectNames", "").split(", ")) - assert expected_names == reported_names, f"Object names mismatch for {expected_obj['ObjectType']}" + if "Details" in expected_obj and "Details" in reported_obj: + assert expected_obj["Details"] == reported_obj["Details"], f"Details mismatch for {expected_obj['ObjectType']}" + + expected_names = sorted(expected_obj.get("ObjectNames", "").split(", ")) + reported_names = sorted(reported_obj.get("ObjectNames", "").split(", ")) + assert expected_names == reported_names, f"Object names mismatch for {expected_obj['ObjectType']}" def validate_report_issues(report, expected_issues): # FilePath reported in the report can be different depending on the machine diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index ceecd2595a..1ed87328e6 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -46,7 +46,7 @@ { "ObjectType": "TABLE", "TotalCount": 68, - "InvalidCount": 5, + "InvalidCount": 68, "ObjectNames": "humanresources.department, humanresources.employee, humanresources.employeedepartmenthistory, humanresources.employeepayhistory, humanresources.jobcandidate, humanresources.shift, person.address, person.businessentityaddress, person.countryregion, person.emailaddress, person.person, person.personphone, person.phonenumbertype, person.stateprovince, person.addresstype, person.businessentity, person.businessentitycontact, person.contacttype, person.password, production.billofmaterials, production.culture, production.document, production.illustration, production.location, production.product, production.productcategory, production.productcosthistory, production.productdescription, production.productdocument, production.productinventory, production.productlistpricehistory, production.productmodel, production.productmodelillustration, production.productmodelproductdescriptionculture, production.productphoto, production.productproductphoto, production.productreview, production.productsubcategory, production.scrapreason, production.transactionhistory, production.transactionhistoryarchive, production.unitmeasure, production.workorder, production.workorderrouting, purchasing.purchaseorderdetail, purchasing.purchaseorderheader, purchasing.productvendor, purchasing.shipmethod, purchasing.vendor, sales.customer, sales.creditcard, sales.currencyrate, sales.countryregioncurrency, sales.currency, sales.personcreditcard, sales.store, sales.shoppingcartitem, sales.specialoffer, sales.salesorderdetail, sales.salesorderheader, sales.salesorderheadersalesreason, sales.specialofferproduct, sales.salesperson, sales.salespersonquotahistory, sales.salesreason, sales.salesterritory, sales.salesterritoryhistory, sales.salestaxrate" }, { diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index 728396eb1c..fa0a6e7955 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -46,8 +46,7 @@ { "ObjectType": "TABLE", "TotalCount": 68, - "InvalidCount": 0, - "InvalidCount": 5, + "InvalidCount": 68, "ObjectNames": "humanresources.department, humanresources.employee, humanresources.employeedepartmenthistory, humanresources.employeepayhistory, humanresources.jobcandidate, humanresources.shift, person.address, person.businessentityaddress, person.countryregion, person.emailaddress, person.person, person.personphone, person.phonenumbertype, person.stateprovince, person.addresstype, person.businessentity, person.businessentitycontact, person.contacttype, person.password, production.billofmaterials, production.culture, production.document, production.illustration, production.location, production.product, production.productcategory, production.productcosthistory, production.productdescription, production.productdocument, production.productinventory, production.productlistpricehistory, production.productmodel, production.productmodelillustration, production.productmodelproductdescriptionculture, production.productphoto, production.productproductphoto, production.productreview, production.productsubcategory, production.scrapreason, production.transactionhistory, production.transactionhistoryarchive, production.unitmeasure, production.workorder, production.workorderrouting, purchasing.purchaseorderdetail, purchasing.purchaseorderheader, purchasing.productvendor, purchasing.shipmethod, purchasing.vendor, sales.customer, sales.creditcard, sales.currencyrate, sales.countryregioncurrency, sales.currency, sales.personcreditcard, sales.store, sales.shoppingcartitem, sales.specialoffer, sales.salesorderdetail, sales.salesorderheader, sales.salesorderheadersalesreason, sales.specialofferproduct, sales.salesperson, sales.salespersonquotahistory, sales.salesreason, sales.salesterritory, sales.salesterritoryhistory, sales.salestaxrate" }, { diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 6021709791..c5171c5d74 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -80,7 +80,7 @@ { "ObjectType": "TRIGGER", "TotalCount": 3, - "InvalidCount": 1, + "InvalidCount": 3, "ObjectNames": "audit_trigger ON public.tt, before_sales_region_insert_update ON public.sales_region, audit_trigger ON schema2.tt" }, { diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 77a3e8bdfe..bb9c5d74a8 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -24,7 +24,7 @@ { "ObjectType": "TABLE", "TotalCount": 175, - "InvalidCount": 0, + "InvalidCount": 62, "ObjectNames": "mgd.acc_accession, mgd.acc_accessionmax, mgd.acc_accessionreference, mgd.acc_actualdb, mgd.acc_logicaldb, mgd.acc_mgitype, mgd.all_allele, mgd.all_allele_cellline, mgd.all_cellline, mgd.all_cellline_derivation, mgd.mgi_user, mgd.prb_strain, mgd.voc_term, mgd.mgi_organism, mgd.mgi_relationship, mgd.mrk_marker, mgd.all_allele_mutation, mgd.voc_annot, mgd.bib_citation_cache, mgd.gxd_allelepair, mgd.voc_annottype, mgd.all_cre_cache, mgd.all_knockout_cache, mgd.all_label, mgd.gxd_allelegenotype, mgd.mgi_reference_assoc, mgd.all_variant, mgd.all_variant_sequence, mgd.bib_refs, mgd.gxd_expression, mgd.gxd_index, mgd.img_image, mgd.mgi_refassoctype, mgd.mgi_synonym, mgd.mgi_synonymtype, mgd.mld_expts, mgd.mld_notes, mgd.mrk_do_cache, mgd.mrk_reference, mgd.mrk_strainmarker, mgd.prb_reference, mgd.prb_source, mgd.voc_evidence, mgd.bib_books, mgd.bib_workflow_status, mgd.bib_notes, mgd.gxd_assay, mgd.gxd_specimen, mgd.bib_workflow_data, mgd.bib_workflow_relevance, mgd.bib_workflow_tag, mgd.crs_cross, mgd.crs_matrix, mgd.crs_progeny, mgd.crs_references, mgd.crs_typings, mgd.dag_closure, mgd.dag_dag, mgd.dag_edge, mgd.dag_label, mgd.dag_node, mgd.voc_vocabdag, mgd.go_tracking, mgd.gxd_antibody, mgd.gxd_antigen, mgd.gxd_antibodyalias, mgd.gxd_antibodymarker, mgd.gxd_antibodyprep, mgd.gxd_gellane, mgd.gxd_insituresult, mgd.gxd_insituresultimage, mgd.gxd_assaytype, mgd.gxd_assaynote, mgd.gxd_gelband, mgd.gxd_gelrow, mgd.gxd_genotype, mgd.gxd_gellanestructure, mgd.gxd_theilerstage, mgd.voc_annotheader, mgd.gxd_htexperiment, mgd.gxd_htexperimentvariable, mgd.gxd_htrawsample, mgd.gxd_htsample, mgd.gxd_htsample_rnaseq, mgd.gxd_htsample_rnaseqcombined, mgd.gxd_htsample_rnaseqset, mgd.gxd_htsample_rnaseqset_cache, mgd.gxd_htsample_rnaseqsetmember, mgd.gxd_index_stages, mgd.gxd_isresultcelltype, mgd.img_imagepane, mgd.gxd_isresultstructure, mgd.gxd_probeprep, mgd.prb_probe, mgd.img_imagepane_assoc, mgd.mgi_note, mgd.map_coord_collection, mgd.map_coord_feature, mgd.map_coordinate, mgd.mrk_chromosome, mgd.mgi_dbinfo, mgd.mgi_keyvalue, mgd.mgi_notetype, mgd.mgi_organism_mgitype, mgd.mgi_property, mgd.mgi_propertytype, mgd.mgi_relationship_category, mgd.mgi_relationship_property, mgd.mgi_set, mgd.mgi_setmember, mgd.mgi_setmember_emapa, mgd.mgi_translation, mgd.mgi_translationtype, mgd.voc_vocab, mgd.mld_assay_types, mgd.mld_concordance, mgd.mld_contig, mgd.mld_contigprobe, mgd.mld_expt_marker, mgd.mld_expt_notes, mgd.mld_fish, mgd.mld_fish_region, mgd.mld_hit, mgd.mld_hybrid, mgd.mld_insitu, mgd.mld_isregion, mgd.mld_matrix, mgd.mld_mc2point, mgd.mld_mcdatalist, mgd.mld_ri, mgd.mld_ri2point, mgd.mld_ridata, mgd.mld_statistics, mgd.mrk_types, mgd.mrk_biotypemapping, mgd.mrk_cluster, mgd.mrk_clustermember, mgd.mrk_current, mgd.mrk_history, mgd.mrk_label, mgd.mrk_location_cache, mgd.mrk_status, mgd.mrk_mcv_cache, mgd.mrk_mcv_count_cache, mgd.mrk_notes, mgd.prb_alias, mgd.prb_allele, mgd.prb_allele_strain, mgd.prb_marker, mgd.prb_notes, mgd.prb_ref_notes, mgd.prb_rflv, mgd.prb_tissue, mgd.prb_strain_genotype, mgd.prb_strain_marker, mgd.ri_riset, mgd.ri_summary, mgd.ri_summary_expt_ref, mgd.seq_allele_assoc, mgd.seq_coord_cache, mgd.seq_genemodel, mgd.seq_genetrap, mgd.seq_marker_cache, mgd.seq_probe_cache, mgd.seq_sequence, mgd.seq_sequence_assoc, mgd.seq_sequence_raw, mgd.seq_source_assoc, mgd.voc_allele_cache, mgd.voc_annot_count_cache, mgd.voc_evidence_property, mgd.voc_marker_cache, mgd.voc_term_emapa, mgd.voc_term_emaps, mgd.wks_rosetta" }, { diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index eb4e17c785..cfe0307c16 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -24,7 +24,7 @@ { "ObjectType": "TABLE", "TotalCount": 175, - "InvalidCount": 0, + "InvalidCount": 62, "ObjectNames": "mgd.acc_accession, mgd.acc_accessionmax, mgd.acc_accessionreference, mgd.acc_actualdb, mgd.acc_logicaldb, mgd.acc_mgitype, mgd.all_allele, mgd.all_allele_cellline, mgd.all_cellline, mgd.all_cellline_derivation, mgd.mgi_user, mgd.prb_strain, mgd.voc_term, mgd.mgi_organism, mgd.mgi_relationship, mgd.mrk_marker, mgd.all_allele_mutation, mgd.voc_annot, mgd.bib_citation_cache, mgd.gxd_allelepair, mgd.voc_annottype, mgd.all_cre_cache, mgd.all_knockout_cache, mgd.all_label, mgd.gxd_allelegenotype, mgd.mgi_reference_assoc, mgd.all_variant, mgd.all_variant_sequence, mgd.bib_refs, mgd.gxd_expression, mgd.gxd_index, mgd.img_image, mgd.mgi_refassoctype, mgd.mgi_synonym, mgd.mgi_synonymtype, mgd.mld_expts, mgd.mld_notes, mgd.mrk_do_cache, mgd.mrk_reference, mgd.mrk_strainmarker, mgd.prb_reference, mgd.prb_source, mgd.voc_evidence, mgd.bib_books, mgd.bib_workflow_status, mgd.bib_notes, mgd.gxd_assay, mgd.gxd_specimen, mgd.bib_workflow_data, mgd.bib_workflow_relevance, mgd.bib_workflow_tag, mgd.crs_cross, mgd.crs_matrix, mgd.crs_progeny, mgd.crs_references, mgd.crs_typings, mgd.dag_closure, mgd.dag_dag, mgd.dag_edge, mgd.dag_label, mgd.dag_node, mgd.voc_vocabdag, mgd.go_tracking, mgd.gxd_antibody, mgd.gxd_antigen, mgd.gxd_antibodyalias, mgd.gxd_antibodymarker, mgd.gxd_antibodyprep, mgd.gxd_gellane, mgd.gxd_insituresult, mgd.gxd_insituresultimage, mgd.gxd_assaytype, mgd.gxd_assaynote, mgd.gxd_gelband, mgd.gxd_gelrow, mgd.gxd_genotype, mgd.gxd_gellanestructure, mgd.gxd_theilerstage, mgd.voc_annotheader, mgd.gxd_htexperiment, mgd.gxd_htexperimentvariable, mgd.gxd_htrawsample, mgd.gxd_htsample, mgd.gxd_htsample_rnaseq, mgd.gxd_htsample_rnaseqcombined, mgd.gxd_htsample_rnaseqset, mgd.gxd_htsample_rnaseqset_cache, mgd.gxd_htsample_rnaseqsetmember, mgd.gxd_index_stages, mgd.gxd_isresultcelltype, mgd.img_imagepane, mgd.gxd_isresultstructure, mgd.gxd_probeprep, mgd.prb_probe, mgd.img_imagepane_assoc, mgd.mgi_note, mgd.map_coord_collection, mgd.map_coord_feature, mgd.map_coordinate, mgd.mrk_chromosome, mgd.mgi_dbinfo, mgd.mgi_keyvalue, mgd.mgi_notetype, mgd.mgi_organism_mgitype, mgd.mgi_property, mgd.mgi_propertytype, mgd.mgi_relationship_category, mgd.mgi_relationship_property, mgd.mgi_set, mgd.mgi_setmember, mgd.mgi_setmember_emapa, mgd.mgi_translation, mgd.mgi_translationtype, mgd.voc_vocab, mgd.mld_assay_types, mgd.mld_concordance, mgd.mld_contig, mgd.mld_contigprobe, mgd.mld_expt_marker, mgd.mld_expt_notes, mgd.mld_fish, mgd.mld_fish_region, mgd.mld_hit, mgd.mld_hybrid, mgd.mld_insitu, mgd.mld_isregion, mgd.mld_matrix, mgd.mld_mc2point, mgd.mld_mcdatalist, mgd.mld_ri, mgd.mld_ri2point, mgd.mld_ridata, mgd.mld_statistics, mgd.mrk_types, mgd.mrk_biotypemapping, mgd.mrk_cluster, mgd.mrk_clustermember, mgd.mrk_current, mgd.mrk_history, mgd.mrk_label, mgd.mrk_location_cache, mgd.mrk_status, mgd.mrk_mcv_cache, mgd.mrk_mcv_count_cache, mgd.mrk_notes, mgd.prb_alias, mgd.prb_allele, mgd.prb_allele_strain, mgd.prb_marker, mgd.prb_notes, mgd.prb_ref_notes, mgd.prb_rflv, mgd.prb_tissue, mgd.prb_strain_genotype, mgd.prb_strain_marker, mgd.ri_riset, mgd.ri_summary, mgd.ri_summary_expt_ref, mgd.seq_allele_assoc, mgd.seq_coord_cache, mgd.seq_genemodel, mgd.seq_genetrap, mgd.seq_marker_cache, mgd.seq_probe_cache, mgd.seq_sequence, mgd.seq_sequence_assoc, mgd.seq_sequence_raw, mgd.seq_source_assoc, mgd.voc_allele_cache, mgd.voc_annot_count_cache, mgd.voc_evidence_property, mgd.voc_marker_cache, mgd.voc_term_emapa, mgd.voc_term_emaps, mgd.wks_rosetta" }, { diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index c569bd9c0f..1050fff604 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -388,6 +388,7 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { reportCase(fpath, "RANGE with offset PRECEDING/FOLLOWING is not supported for column type numeric and offset type double precision", "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true } else if stmt := fetchRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { location := strings.ToUpper(stmt[1]) if slices.Contains(notSupportedFetchLocation, location) { @@ -396,24 +397,31 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } } else if stmt := alterAggRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { + summaryMap["AGGREGATE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER AGGREGATE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropCollRegex.MatchString(sqlInfo.stmt) { + summaryMap["COLLATION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropIdxRegex.MatchString(sqlInfo.stmt) { + summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropViewRegex.MatchString(sqlInfo.stmt) { + summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropSeqRegex.MatchString(sqlInfo.stmt) { + summaryMap["SEQUENCE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropForeignRegex.MatchString(sqlInfo.stmt) { + summaryMap["FOREIGN TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if idx := dropIdxConcurRegex.FindStringSubmatch(sqlInfo.stmt); idx != nil { + summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP INDEX CONCURRENTLY not supported yet", "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if currentOfRegex.MatchString(sqlInfo.stmt) { @@ -449,63 +457,83 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { reportCase(fpath, "OIDs are not supported for user tables.", "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE OF not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterSchemaRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET SCHEMA not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if createSchemaRegex.MatchString(sqlInfo.stmt) { + summaryMap["SCHEMA"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE SCHEMA with elements not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterNotOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE NOT OF not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterColumnStatsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STATISTICS not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterColumnStorageRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STORAGE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterColumnResetAttributesRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column RESET (attribute) not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER CONSTRAINT not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := setOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITH OIDS not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := withoutClusterRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITHOUT CLUSTER not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterSetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterIdxRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER INDEX SET not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterResetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE RESET not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterOptionsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if typ := dropAttrRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { + summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE DROP ATTRIBUTE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if typ := alterTypeRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { + summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterInhRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE INHERIT not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := valConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE VALIDATE CONSTRAINT not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if spc := alterTblSpcRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { + summaryMap["TABLESPACE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLESPACE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if spc := alterViewRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { + summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER VIEW not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := cLangRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { @@ -544,9 +572,11 @@ func checkForeign(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { //TODO: refactor it later to remove all the unneccessary regexes if tbl := primRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Primary key constraints are not supported on foreign tables.", "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := foreignKeyRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Foreign key constraints are not supported on foreign tables.", "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } @@ -619,15 +649,44 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil issueType = UNSUPPORTED_PLPGSQL_OBEJCTS } - //TODO: how to different between same issue on differnt obejct types like ALTER/INDEX for not adding it ot invalid count map - increaseInvalidCount, ok := issueInstance.Details["INCREASE_INVALID_COUNT"] - if !ok || (increaseInvalidCount.(bool)) { - summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true + var constraintIssues = []string{ + queryissue.EXCLUSION_CONSTRAINTS, + queryissue.DEFERRABLE_CONSTRAINTS, + queryissue.PK_UK_ON_COMPLEX_DATATYPE, + } + /* + TODO: + // unsupportedIndexIssue + // ObjectType = INDEX + // ObjectName = idx_name ON table_name + // invalidCount.Type = INDEX + // invalidCount.Name = ObjectName (because this is fully qualified) + // DisplayName = ObjectName + + // deferrableConstraintIssue + // ObjectType = TABLE + // ObjectName = table_name + // invalidCount.Type = TABLE + // invalidCount.Name = ObjectName + // DisplayName = table_name (constraint_name) (!= ObjectName) + + // Solutions + // 1. Define a issue.ObjectDisplayName + // 2. Keep it in issue.Details and write logic in UI layer to construct display name. + */ + displayObjectName := issueInstance.ObjectName + + constraintName, ok := issueInstance.Details[queryissue.CONSTRAINT_NAME] + if slices.Contains(constraintIssues, issueInstance.Type) && ok { + //In case of constraint issues we add constraint name to the object name as well + displayObjectName = fmt.Sprintf("%s, constraint: (%s)", issueInstance.ObjectName, constraintName) } + summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true + return utils.Issue{ ObjectType: issueInstance.ObjectType, - ObjectName: issueInstance.ObjectName, + ObjectName: displayObjectName, Reason: issueInstance.TypeName, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 67d95e33c1..215f055efd 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -101,7 +101,7 @@
    Feature Name
    - + {{range .SchemaSummary.DBObjects}} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.html b/yb-voyager/cmd/templates/schema_analysis_report.html index 34a4b6e230..562ee4f50d 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.html +++ b/yb-voyager/cmd/templates/schema_analysis_report.html @@ -80,7 +80,7 @@

    Migration Information

    Schema Summary

    Object TypeTotal CountTotal Objects Object Names
    - + {{ range .SchemaSummary.DBObjects }} {{ if .TotalCount }} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.txt b/yb-voyager/cmd/templates/schema_analysis_report.txt index 2c1bcfffa8..2d046227bf 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.txt +++ b/yb-voyager/cmd/templates/schema_analysis_report.txt @@ -19,12 +19,12 @@ Migration Complexity : {{ .MigrationComplexity }} Schema Summary --------------- {{ range .SchemaSummary.DBObjects }} -Object Type : {{ .ObjectType }} - - Total Count : {{ .TotalCount }} - - Valid Count : {{ sub .TotalCount .InvalidCount }} - - Invalid Count : {{ .InvalidCount }} - - Object Names : {{ .ObjectNames }}{{ if .Details }} - - Details : {{ .Details }} +Object Type : {{ .ObjectType }} + - Total Objects : {{ .TotalCount }} + - Objects Without Issues : {{ sub .TotalCount .InvalidCount }} + - Objects With Issues : {{ .InvalidCount }} + - Object Names : {{ .ObjectNames }}{{ if .Details }} + - Details : {{ .Details }} {{ end }} {{ end }} diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 49fdf5423d..dbab60c240 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -56,6 +56,7 @@ const ( // Object types const ( + CONSTRAINT_NAME = "ConstraintName" TABLE_OBJECT_TYPE = "TABLE" FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" FUNCTION_OBJECT_TYPE = "FUNCTION" diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 33714d00c6..a791e96066 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -80,16 +80,18 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if c.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { issues = append(issues, NewExclusionConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + table.GetObjectName(), "", + c.ConstraintName, )) } if c.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && c.IsDeferrable { issues = append(issues, NewDeferrableConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + table.GetObjectName(), "", + c.ConstraintName, )) } @@ -106,10 +108,10 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss } issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + table.GetObjectName(), "", typeName, - true, + c.ConstraintName, )) } } @@ -413,15 +415,17 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q if alter.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { issues = append(issues, NewExclusionConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + alter.GetObjectName(), "", + alter.ConstraintName, )) } if alter.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && alter.IsDeferrable { issues = append(issues, NewDeferrableConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + alter.GetObjectName(), "", + alter.ConstraintName, )) } @@ -447,10 +451,10 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q } issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + alter.GetObjectName(), "", typeName, - false, + alter.ConstraintName, )) } diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 08fe2c69c1..b131eb62b1 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -51,10 +51,6 @@ var unloggedTableIssue = issue.Issue{ func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for UNLOGGED TABLE as its not reported in the TABLE objects - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(unloggedTableIssue, objectType, objectName, sqlStatement, details) } @@ -81,10 +77,6 @@ var storageParameterIssue = issue.Issue{ func NewStorageParameterIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(storageParameterIssue, objectType, objectName, sqlStatement, details) } @@ -98,10 +90,6 @@ var setAttributeIssue = issue.Issue{ func NewSetAttributeIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(setAttributeIssue, objectType, objectName, sqlStatement, details) } @@ -115,10 +103,6 @@ var clusterOnIssue = issue.Issue{ func NewClusterONIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(clusterOnIssue, objectType, objectName, sqlStatement, details) } @@ -132,10 +116,6 @@ var disableRuleIssue = issue.Issue{ func NewDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } issue := disableRuleIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, ruleName) return newQueryIssue(issue, objectType, objectName, sqlStatement, details) @@ -149,8 +129,11 @@ var exclusionConstraintIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", } -func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string) QueryIssue { - return newQueryIssue(exclusionConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, + } + return newQueryIssue(exclusionConstraintIssue, objectType, objectName, sqlStatement, details) } var deferrableConstraintIssue = issue.Issue{ @@ -161,8 +144,11 @@ var deferrableConstraintIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", } -func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string) QueryIssue { - return newQueryIssue(deferrableConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, + } + return newQueryIssue(deferrableConstraintIssue, objectType, objectName, sqlStatement, details) } var multiColumnGinIndexIssue = issue.Issue{ @@ -195,10 +181,10 @@ var policyRoleIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", } -func NewPolicyRoleIssue(objectType string, objectName string, SqlStatement string, roles []string) QueryIssue { +func NewPolicyRoleIssue(objectType string, objectName string, sqlStatement string, roles []string) QueryIssue { issue := policyRoleIssue issue.TypeName = fmt.Sprintf("%s Users - (%s)", issue.TypeName, strings.Join(roles, ",")) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var constraintTriggerIssue = issue.Issue{ @@ -208,13 +194,9 @@ var constraintTriggerIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", } -func NewConstraintTriggerIssue(objectType string, objectName string, SqlStatement string) QueryIssue { +func NewConstraintTriggerIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for CONSTRAINT TRIGGER we don't have separate object type TODO: fix - if objectType == "TRIGGER" { - details["INCREASE_INVALID_COUNT"] = false - } - return newQueryIssue(constraintTriggerIssue, objectType, objectName, SqlStatement, details) + return newQueryIssue(constraintTriggerIssue, objectType, objectName, sqlStatement, details) } var referencingClauseInTriggerIssue = issue.Issue{ @@ -224,8 +206,8 @@ var referencingClauseInTriggerIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", } -func NewReferencingClauseTrigIssue(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(referencingClauseInTriggerIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewReferencingClauseTrigIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(referencingClauseInTriggerIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ @@ -236,8 +218,8 @@ var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ Suggestion: "Create the triggers on individual partitions.", } -func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var alterTableAddPKOnPartitionIssue = issue.Issue{ @@ -252,13 +234,9 @@ var alterTableAddPKOnPartitionIssue = issue.Issue{ }, } -func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { +func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } - return newQueryIssue(alterTableAddPKOnPartitionIssue, objectType, objectName, SqlStatement, details) + return newQueryIssue(alterTableAddPKOnPartitionIssue, objectType, objectName, sqlStatement, details) } var expressionPartitionIssue = issue.Issue{ @@ -269,8 +247,8 @@ var expressionPartitionIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", } -func NewExpressionPartitionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(expressionPartitionIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewExpressionPartitionIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(expressionPartitionIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var multiColumnListPartition = issue.Issue{ @@ -281,8 +259,8 @@ var multiColumnListPartition = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", } -func NewMultiColumnListPartition(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(multiColumnListPartition, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewMultiColumnListPartition(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(multiColumnListPartition, objectType, objectName, sqlStatement, map[string]interface{}{}) } var insufficientColumnsInPKForPartition = issue.Issue{ @@ -293,10 +271,10 @@ var insufficientColumnsInPKForPartition = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", } -func NewInsufficientColumnInPKForPartition(objectType string, objectName string, SqlStatement string, partitionColumnsNotInPK []string) QueryIssue { +func NewInsufficientColumnInPKForPartition(objectType string, objectName string, sqlStatement string, partitionColumnsNotInPK []string) QueryIssue { issue := insufficientColumnsInPKForPartition issue.TypeName = fmt.Sprintf("%s - (%s)", issue.TypeName, strings.Join(partitionColumnsNotInPK, ", ")) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xmlDatatypeIssue = issue.Issue{ @@ -307,10 +285,10 @@ var xmlDatatypeIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", } -func NewXMLDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { +func NewXMLDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xmlDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xidDatatypeIssue = issue.Issue{ @@ -321,10 +299,10 @@ var xidDatatypeIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", } -func NewXIDDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { +func NewXIDDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xidDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var postgisDatatypeIssue = issue.Issue{ @@ -334,10 +312,10 @@ var postgisDatatypeIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } -func NewPostGisDatatypeIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewPostGisDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := postgisDatatypeIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesIssue = issue.Issue{ @@ -347,10 +325,10 @@ var unsupportedDatatypesIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } -func NewUnsupportedDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewUnsupportedDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ @@ -360,10 +338,10 @@ var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } -func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ @@ -373,10 +351,10 @@ var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } -func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationWithFFOrFBIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ @@ -387,15 +365,13 @@ var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, } -func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, SqlStatement string, typeName string, increaseInvalidCnt bool) QueryIssue { - details := map[string]interface{}{} - //for ALTER not increasing count, but for Create increasing TODO: fix - if !increaseInvalidCnt { - details["INCREASE_INVALID_COUNT"] = false +func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, sqlStatement string, typeName string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, } issue := primaryOrUniqueOnUnsupportedIndexTypesIssue issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, details) + return newQueryIssue(issue, objectType, objectName, sqlStatement, details) } var indexOnComplexDatatypesIssue = issue.Issue{ @@ -406,10 +382,10 @@ var indexOnComplexDatatypesIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", } -func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string) QueryIssue { +func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string) QueryIssue { issue := indexOnComplexDatatypesIssue issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var foreignTableIssue = issue.Issue{ @@ -420,10 +396,10 @@ var foreignTableIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", } -func NewForeignTableIssue(objectType string, objectName string, SqlStatement string, serverName string) QueryIssue { +func NewForeignTableIssue(objectType string, objectName string, sqlStatement string, serverName string) QueryIssue { issue := foreignTableIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, serverName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var inheritanceIssue = issue.Issue{ diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 99d989b528..cf2a95eedb 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -191,7 +191,7 @@ func TestAllIssues(t *testing.T) { NewDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), }, stmt5: []QueryIssue{ - NewDeferrableConstraintIssue("TABLE", "abc, constraint: (cnstr_id)", stmt5), + NewDeferrableConstraintIssue("TABLE", "abc", stmt5, "cnstr_id"), }, stmt6: []QueryIssue{ NewAdvisoryLocksIssue("DML_QUERY", "", stmt6), @@ -210,8 +210,8 @@ func TestAllIssues(t *testing.T) { NewInsufficientColumnInPKForPartition("TABLE", "test_non_pk_multi_column_list", stmt10, []string{"country_code", "record_type"}), }, stmt11: []QueryIssue{ - NewExclusionConstraintIssue("TABLE", "Test, constraint: (Test_room_id_time_range_excl)", stmt11), - NewExclusionConstraintIssue("TABLE", "Test, constraint: (no_time_overlap_constr)", stmt11), + NewExclusionConstraintIssue("TABLE", "Test", stmt11, "Test_room_id_time_range_excl"), + NewExclusionConstraintIssue("TABLE", "Test", stmt11, "no_time_overlap_constr"), }, stmt13: []QueryIssue{ NewIndexOnComplexDatatypesIssue("INDEX", "idx_on_daterange ON test_dt", stmt13, "daterange"), @@ -259,7 +259,7 @@ func TestDDLIssues(t *testing.T) { }, stmt16: []QueryIssue{ NewXmlFunctionsIssue("TABLE", "public.xml_data_example", stmt16), - NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue("TABLE", "public.xml_data_example, constraint: (xml_data_example_d_key)", stmt16, "daterange", true), + NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue("TABLE", "public.xml_data_example", stmt16, "daterange", "xml_data_example_d_key"), NewMultiColumnListPartition("TABLE", "public.xml_data_example", stmt16), NewInsufficientColumnInPKForPartition("TABLE", "public.xml_data_example", stmt16, []string{"name"}), NewXMLDatatypeIssue("TABLE", "public.xml_data_example", stmt16, "description"), diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index c2b0abb8c9..4c18b78fad 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -90,6 +90,7 @@ type SchemaSummary struct { DBObjects []DBObject `json:"DatabaseObjects"` } +//TODO: Rename the variables of TotalCount and InvalidCount -> TotalObjects and ObjectsWithIssues type DBObject struct { ObjectType string `json:"ObjectType"` TotalCount int `json:"TotalCount"` From 5137a39c8c93e9b7416eba617169f264ee558b82 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 18 Dec 2024 22:05:59 +0530 Subject: [PATCH 065/105] [DB-14414] Filtering queries based on tables schema in the query for unsupported query constructs (#2099) * For reporting unsupported query constructs considering filtering queries based on tablenames in it, ignore function names * Add end to end test for testing schema filtering in unsupported query constructs * Add a constants package for db types etc in other packages under src directory --- .github/workflows/misc-migtests.yml | 3 + .../pg/assessment-report-test-uqc/cleanup-db | 10 + .../pg/assessment-report-test-uqc/env.sh | 3 + .../expectedAssessmentReport.json | 237 ++++++++++++++++++ .../pg/assessment-report-test-uqc/fix-schema | 0 .../pg/assessment-report-test-uqc/init-db | 18 ++ .../pg_assessment_report_uqc.sql | 44 ++++ .../unsupported_query_constructs.sql | 23 ++ yb-voyager/src/constants/constants.go | 28 +++ yb-voyager/src/namereg/namereg.go | 28 +-- yb-voyager/src/namereg/namereg_test.go | 29 +-- .../src/query/queryissue/detectors_test.go | 2 +- .../src/query/queryparser/object_collector.go | 39 ++- .../src/query/queryparser/traversal_proto.go | 2 +- .../src/query/queryparser/traversal_test.go | 134 +++++++++- .../src/tgtdb/attr_name_registry_test.go | 5 +- yb-voyager/src/tgtdb/oracle.go | 3 +- yb-voyager/src/tgtdb/postgres.go | 3 +- yb-voyager/src/tgtdb/yugabytedb.go | 3 +- yb-voyager/src/utils/sqlname/nametuple.go | 10 +- .../src/utils/sqlname/nametuple_test.go | 33 +-- yb-voyager/src/utils/sqlname/sqlname.go | 48 ++-- 22 files changed, 608 insertions(+), 97 deletions(-) create mode 100755 migtests/tests/pg/assessment-report-test-uqc/cleanup-db create mode 100644 migtests/tests/pg/assessment-report-test-uqc/env.sh create mode 100644 migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json create mode 100755 migtests/tests/pg/assessment-report-test-uqc/fix-schema create mode 100755 migtests/tests/pg/assessment-report-test-uqc/init-db create mode 100644 migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql create mode 100644 migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql create mode 100644 yb-voyager/src/constants/constants.go diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index cb05a9fbb5..add010cdd9 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -72,6 +72,9 @@ jobs: - name: "TEST: Assessment Report Test" run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test + + - name: "TEST: Assessment Report Test (Schema list UQC)" + run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test-uqc - name: "TEST: analyze-schema" if: ${{ !cancelled() }} diff --git a/migtests/tests/pg/assessment-report-test-uqc/cleanup-db b/migtests/tests/pg/assessment-report-test-uqc/cleanup-db new file mode 100755 index 0000000000..2793da7cec --- /dev/null +++ b/migtests/tests/pg/assessment-report-test-uqc/cleanup-db @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e +set -x + +source ${SCRIPTS}/functions.sh + + +echo "Deleting ${SOURCE_DB_NAME} database on source" +run_psql postgres "DROP DATABASE ${SOURCE_DB_NAME};" \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/env.sh b/migtests/tests/pg/assessment-report-test-uqc/env.sh new file mode 100644 index 0000000000..0d09ece1c9 --- /dev/null +++ b/migtests/tests/pg/assessment-report-test-uqc/env.sh @@ -0,0 +1,3 @@ +export SOURCE_DB_TYPE="postgresql" +export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"pg_assessment_report_uqc"} +export SOURCE_DB_SCHEMA="sales,analytics" diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json new file mode 100644 index 0000000000..720906972f --- /dev/null +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -0,0 +1,237 @@ +{ + "VoyagerVersion": "IGNORED", + "TargetDBVersion": "IGNORED", + "MigrationComplexity": "LOW", + "SchemaSummary": { + "Description": "Objects that will be created on the target YugabyteDB.", + "DbName": "pg_assessment_report_uqc", + "SchemaNames": [ + "sales", + "analytics" + ], + "DbVersion": "14.13 (Ubuntu 14.13-1.pgdg20.04+1)", + "DatabaseObjects": [ + { + "ObjectType": "SCHEMA", + "TotalCount": 2, + "InvalidCount": 0, + "ObjectNames": "analytics, sales" + }, + { + "ObjectType": "EXTENSION", + "TotalCount": 1, + "InvalidCount": 0, + "ObjectNames": "pg_stat_statements" + }, + { + "ObjectType": "TABLE", + "TotalCount": 2, + "InvalidCount": 0, + "ObjectNames": "analytics.metrics, sales.orders" + } + ] + }, + "Sizing": { + "SizingRecommendation": { + "ColocatedTables": [ + "sales.orders", + "analytics.metrics" + ], + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 2 objects (2 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ShardedTables": null, + "NumNodes": 3, + "VCPUsPerInstance": 4, + "MemoryPerInstance": 16, + "OptimalSelectConnectionsPerNode": 8, + "OptimalInsertConnectionsPerNode": 12, + "EstimatedTimeInMinForImport": 1, + "ParallelVoyagerJobs": 1 + }, + "FailureReasoning": "" + }, + "UnsupportedDataTypes": null, + "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", + "UnsupportedFeatures": [ + { + "FeatureName": "GIST indexes", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "BRIN indexes", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "SPGIST indexes", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Constraint triggers", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Inherited tables", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Tables with stored generated columns", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Conversion objects", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Gin indexes on multi-columns", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Setting attribute=value on column", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Disabling rule on table", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Clustering table on index", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Storage parameters in DDLs", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Extensions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Exclusion constraints", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Deferrable constraints", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "View with check option", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Index on complex datatypes", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Primary / Unique key constraints on complex datatypes", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Unlogged tables", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "REFERENCING clause for triggers", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "BEFORE ROW triggers on Partitioned tables", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Advisory Locks", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "XML Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "System Columns", + "Objects": [], + "MinimumVersionsFixedIn": null + } + ], + "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", + "TableIndexStats": [ + { + "SchemaName": "sales", + "ObjectName": "orders", + "RowCount": 2, + "ColumnCount": 3, + "Reads": 0, + "Writes": 2, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, + { + "SchemaName": "analytics", + "ObjectName": "metrics", + "RowCount": 2, + "ColumnCount": 3, + "Reads": 2, + "Writes": 2, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + } + ], + "Notes": null, + "MigrationCaveats": [ + { + "FeatureName": "Alter partitioned tables to add Primary Key", + "Objects": [], + "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Foreign tables", + "Objects": [], + "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Policies", + "Objects": [], + "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", + "MinimumVersionsFixedIn": null + } + ], + "UnsupportedQueryConstructs": [ + { + "ConstructTypeName": "Advisory Locks", + "Query": "SELECT metric_name, pg_advisory_lock(metric_id)\nFROM analytics.metrics\nWHERE metric_value \u003e $1", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + "MinimumVersionsFixedIn": null + } + ], + "UnsupportedPlPgSqlObjects": null +} \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/fix-schema b/migtests/tests/pg/assessment-report-test-uqc/fix-schema new file mode 100755 index 0000000000..e69de29bb2 diff --git a/migtests/tests/pg/assessment-report-test-uqc/init-db b/migtests/tests/pg/assessment-report-test-uqc/init-db new file mode 100755 index 0000000000..c34c7fd127 --- /dev/null +++ b/migtests/tests/pg/assessment-report-test-uqc/init-db @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e +set -x + +source ${SCRIPTS}/functions.sh + +echo "Creating ${SOURCE_DB_NAME} database on source" +run_psql postgres "DROP DATABASE IF EXISTS ${SOURCE_DB_NAME};" +run_psql postgres "CREATE DATABASE ${SOURCE_DB_NAME};" + +echo "Initialising source database." + +run_psql "${SOURCE_DB_NAME}" "\i pg_assessment_report_uqc.sql" + +run_psql "${SOURCE_DB_NAME}" "\i unsupported_query_constructs.sql" + +echo "End of init-db script" \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql new file mode 100644 index 0000000000..c4eff19e41 --- /dev/null +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -0,0 +1,44 @@ +-- Create multiple schemas +CREATE SCHEMA public; +CREATE SCHEMA sales; +CREATE SCHEMA hr; +CREATE SCHEMA analytics; + +-- Create tables in each schema +CREATE TABLE public.employees ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT, + department_id INT +); + +CREATE TABLE sales.orders ( + order_id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + customer_id INT, + amount NUMERIC +); + +CREATE TABLE hr.departments ( + department_id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + department_name TEXT, + location TEXT +); + +CREATE TABLE analytics.metrics ( + metric_id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + metric_name TEXT, + metric_value NUMERIC +); + +-- Create a view in public schema +CREATE VIEW public.employee_view AS +SELECT e.id, e.name, d.department_name +FROM public.employees e +JOIN hr.departments d ON e.department_id = d.department_id; + + +-- Insert some sample data +INSERT INTO hr.departments (department_name, location) VALUES ('Engineering', 'Building A'); +INSERT INTO hr.departments (department_name, location) VALUES ('Sales', 'Building B'); +INSERT INTO public.employees (name, department_id) VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 2); +INSERT INTO sales.orders (customer_id, amount) VALUES (101, 500.00), (102, 1200.00); +INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05); \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql new file mode 100644 index 0000000000..a0ebe3d97a --- /dev/null +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -0,0 +1,23 @@ +-- Unsupported Query Constructs +DROP EXTENSION IF EXISTS pg_stat_statements; +CREATE EXTENSION pg_stat_statements; +SELECT pg_stat_statements_reset(); +SELECT * FROM pg_stat_statements; + + +-- 1) System columns usage (public schema) +SELECT name, xmin FROM public.employees WHERE id = 1; + +-- 2) Advisory locks (hr schema) +SELECT hr.departments.department_name, pg_advisory_lock(hr.departments.department_id) +FROM hr.departments +WHERE department_name = 'Engineering'; + +-- 3) XML function usage (public schema) +SELECT xmlelement(name "employee_data", name) AS emp_xml +FROM public.employees; + +-- 4) Advisory locks (analytics schema) +SELECT metric_name, pg_advisory_lock(metric_id) +FROM analytics.metrics +WHERE metric_value > 0.02; \ No newline at end of file diff --git a/yb-voyager/src/constants/constants.go b/yb-voyager/src/constants/constants.go new file mode 100644 index 0000000000..7912fb1d5c --- /dev/null +++ b/yb-voyager/src/constants/constants.go @@ -0,0 +1,28 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package constants + +const ( + // Database Object types + TABLE = "table" + FUNCTION = "function" + + // Source DB Types + YUGABYTEDB = "yugabytedb" + POSTGRESQL = "postgresql" + ORACLE = "oracle" + MYSQL = "mysql" +) diff --git a/yb-voyager/src/namereg/namereg.go b/yb-voyager/src/namereg/namereg.go index 5aebe6380e..f1b7890d3e 100644 --- a/yb-voyager/src/namereg/namereg.go +++ b/yb-voyager/src/namereg/namereg.go @@ -8,18 +8,12 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/jsonfile" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) -const ( - YUGABYTEDB = sqlname.YUGABYTEDB - POSTGRESQL = sqlname.POSTGRESQL - ORACLE = sqlname.ORACLE - MYSQL = sqlname.MYSQL -) - const ( TARGET_DB_IMPORTER_ROLE = "target_db_importer" SOURCE_DB_IMPORTER_ROLE = "source_db_importer" // Fallback. @@ -130,7 +124,7 @@ func (reg *NameRegistry) registerNames() (bool, error) { return reg.registerYBNames() case reg.params.Role == SOURCE_REPLICA_DB_IMPORTER_ROLE && reg.DefaultSourceReplicaDBSchemaName == "": log.Infof("setting default source replica schema name in the name registry: %s", reg.DefaultSourceDBSchemaName) - defaultSchema := lo.Ternary(reg.SourceDBType == POSTGRESQL, reg.DefaultSourceDBSchemaName, reg.params.TargetDBSchema) + defaultSchema := lo.Ternary(reg.SourceDBType == constants.POSTGRESQL, reg.DefaultSourceDBSchemaName, reg.params.TargetDBSchema) reg.setDefaultSourceReplicaDBSchemaName(defaultSchema) return true, nil } @@ -174,11 +168,11 @@ func (reg *NameRegistry) initSourceDBSchemaNames() { // source.Schema contains only one schema name for MySQL and Oracle; whereas // it contains a pipe separated list for postgres. switch reg.params.SourceDBType { - case ORACLE: + case constants.ORACLE: reg.SourceDBSchemaNames = []string{strings.ToUpper(reg.params.SourceDBSchema)} - case MYSQL: + case constants.MYSQL: reg.SourceDBSchemaNames = []string{reg.params.SourceDBName} - case POSTGRESQL: + case constants.POSTGRESQL: reg.SourceDBSchemaNames = lo.Map(strings.Split(reg.params.SourceDBSchema, "|"), func(s string, _ int) string { return strings.ToLower(s) }) @@ -198,11 +192,11 @@ func (reg *NameRegistry) registerYBNames() (bool, error) { m := make(map[string][]string) reg.DefaultYBSchemaName = reg.params.TargetDBSchema - if reg.SourceDBTableNames != nil && reg.SourceDBType == POSTGRESQL { + if reg.SourceDBTableNames != nil && reg.SourceDBType == constants.POSTGRESQL { reg.DefaultYBSchemaName = reg.DefaultSourceDBSchemaName } switch reg.SourceDBType { - case POSTGRESQL: + case constants.POSTGRESQL: reg.YBSchemaNames = reg.SourceDBSchemaNames default: reg.YBSchemaNames = []string{reg.params.TargetDBSchema} @@ -294,9 +288,9 @@ func (reg *NameRegistry) LookupTableName(tableNameArg string) (sqlname.NameTuple if errors.As(err, &errObj) { // Case insensitive match. caseInsensitiveName := tableName - if reg.SourceDBType == POSTGRESQL || reg.SourceDBType == YUGABYTEDB { + if reg.SourceDBType == constants.POSTGRESQL || reg.SourceDBType == constants.YUGABYTEDB { caseInsensitiveName = strings.ToLower(tableName) - } else if reg.SourceDBType == ORACLE { + } else if reg.SourceDBType == constants.ORACLE { caseInsensitiveName = strings.ToUpper(tableName) } if lo.Contains(errObj.Names, caseInsensitiveName) { @@ -312,14 +306,14 @@ func (reg *NameRegistry) LookupTableName(tableNameArg string) (sqlname.NameTuple } if reg.YBTableNames != nil { // nil in `export` mode. targetName, err = reg.lookup( - YUGABYTEDB, reg.YBTableNames, reg.DefaultYBSchemaName, schemaName, tableName) + constants.YUGABYTEDB, reg.YBTableNames, reg.DefaultYBSchemaName, schemaName, tableName) if err != nil { errObj := &ErrMultipleMatchingNames{} if errors.As(err, &errObj) { // A special case. if lo.Contains(errObj.Names, strings.ToLower(tableName)) { targetName, err = reg.lookup( - YUGABYTEDB, reg.YBTableNames, reg.DefaultYBSchemaName, schemaName, strings.ToLower(tableName)) + constants.YUGABYTEDB, reg.YBTableNames, reg.DefaultYBSchemaName, schemaName, strings.ToLower(tableName)) } } if err != nil { diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index 0a0951581b..9d6b16c523 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -12,12 +12,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) var oracleToYBNameRegistry = &NameRegistry{ - SourceDBType: ORACLE, + SourceDBType: constants.ORACLE, params: NameRegistryParams{ Role: TARGET_DB_IMPORTER_ROLE, }, @@ -41,15 +42,15 @@ func buildNameTuple(reg *NameRegistry, sourceSchema, sourceTable, targetSchema, sourceName = sqlname.NewObjectName(reg.SourceDBType, sourceSchema, sourceSchema, sourceTable) } if targetSchema != "" && targetTable != "" { - targetName = sqlname.NewObjectName(YUGABYTEDB, targetSchema, targetSchema, targetTable) + targetName = sqlname.NewObjectName(constants.YUGABYTEDB, targetSchema, targetSchema, targetTable) } return NewNameTuple(reg.params.Role, sourceName, targetName) } func TestNameTuple(t *testing.T) { assert := assert.New(t) - sourceName := sqlname.NewObjectName(ORACLE, "SAKILA", "SAKILA", "TABLE1") - targetName := sqlname.NewObjectName(YUGABYTEDB, "public", "public", "table1") + sourceName := sqlname.NewObjectName(constants.ORACLE, "SAKILA", "SAKILA", "TABLE1") + targetName := sqlname.NewObjectName(constants.YUGABYTEDB, "public", "public", "table1") ntup := NewNameTuple(TARGET_DB_IMPORTER_ROLE, sourceName, targetName) @@ -76,8 +77,8 @@ func TestNameTuple(t *testing.T) { func TestNameTupleMatchesPattern(t *testing.T) { assert := assert.New(t) - sourceName := sqlname.NewObjectName(ORACLE, "SAKILA", "SAKILA", "TABLE1") - targetName := sqlname.NewObjectName(YUGABYTEDB, "public", "sakila", "table1") + sourceName := sqlname.NewObjectName(constants.ORACLE, "SAKILA", "SAKILA", "TABLE1") + targetName := sqlname.NewObjectName(constants.YUGABYTEDB, "public", "sakila", "table1") ntup := NewNameTuple(TARGET_DB_IMPORTER_ROLE, sourceName, targetName) testCases := []struct { @@ -112,8 +113,8 @@ func TestNameTupleMatchesPattern(t *testing.T) { func TestNameTupleMatchesPatternMySQL(t *testing.T) { assert := assert.New(t) - sourceName := sqlname.NewObjectName(MYSQL, "test", "test", "Table1") - targetName := sqlname.NewObjectName(YUGABYTEDB, "public", "test", "table1") + sourceName := sqlname.NewObjectName(constants.MYSQL, "test", "test", "Table1") + targetName := sqlname.NewObjectName(constants.YUGABYTEDB, "public", "test", "table1") ntup := NewNameTuple(TARGET_DB_IMPORTER_ROLE, sourceName, targetName) testCases := []struct { pattern string @@ -150,7 +151,7 @@ func TestNameMatchesPattern(t *testing.T) { require := require.New(t) reg := &NameRegistry{ - SourceDBType: ORACLE, + SourceDBType: constants.ORACLE, params: NameRegistryParams{ Role: SOURCE_DB_EXPORTER_ROLE, }, @@ -472,7 +473,7 @@ func TestNameRegistryWithDummyDBs(t *testing.T) { params := NameRegistryParams{ FilePath: "", Role: currentMode, - SourceDBType: ORACLE, + SourceDBType: constants.ORACLE, SourceDBSchema: "SAKILA", SourceDBName: "ORCLPDB1", TargetDBSchema: tSchema, @@ -490,7 +491,7 @@ func TestNameRegistryWithDummyDBs(t *testing.T) { err := reg.Init() require.Nil(err) - assert.Equal(ORACLE, reg.SourceDBType) + assert.Equal(constants.ORACLE, reg.SourceDBType) assert.Equal("SAKILA", reg.DefaultSourceDBSchemaName) assert.Equal(sourceNamesMap, reg.SourceDBTableNames) table1 := buildNameTuple(reg, "SAKILA", "TABLE1", "", "") @@ -506,7 +507,7 @@ func TestNameRegistryWithDummyDBs(t *testing.T) { reg = newNameRegistry("") err = reg.Init() require.Nil(err) - assert.Equal(ORACLE, reg.SourceDBType) + assert.Equal(constants.ORACLE, reg.SourceDBType) assert.Equal("SAKILA", reg.DefaultSourceDBSchemaName) assert.Equal(sourceNamesMap, reg.SourceDBTableNames) ntup, err = reg.LookupTableName("TABLE1") @@ -606,11 +607,11 @@ func TestNameRegistryJson(t *testing.T) { // Create a sample NameRegistry instance reg := &NameRegistry{ - SourceDBType: ORACLE, + SourceDBType: constants.ORACLE, params: NameRegistryParams{ FilePath: outputFilePath, Role: TARGET_DB_IMPORTER_ROLE, - SourceDBType: ORACLE, + SourceDBType: constants.ORACLE, SourceDBSchema: "SAKILA", SourceDBName: "ORCLPDB1", TargetDBSchema: "ybsakila", diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 2845fc2adb..6320e9ca35 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -652,7 +652,7 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { visited := make(map[protoreflect.Message]bool) - objectCollector := queryparser.NewObjectCollector() + objectCollector := queryparser.NewObjectCollector(nil) processor := func(msg protoreflect.Message) error { for _, detector := range detectors { log.Debugf("running detector %T", detector) diff --git a/yb-voyager/src/query/queryparser/object_collector.go b/yb-voyager/src/query/queryparser/object_collector.go index 62f69c9c34..4da4682652 100644 --- a/yb-voyager/src/query/queryparser/object_collector.go +++ b/yb-voyager/src/query/queryparser/object_collector.go @@ -20,17 +20,39 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "google.golang.org/protobuf/reflect/protoreflect" ) -// ObjectCollector collects unique schema-qualified object names. +// ObjectPredicate defines a function signature that decides whether to include an object. +type ObjectPredicate func(schemaName string, objectName string, objectType string) bool + +// Using a predicate makes it easy to adapt or swap filtering logic in the future without changing the collector. + +// ObjectCollector collects unique schema-qualified object names based on the provided predicate. type ObjectCollector struct { objectSet map[string]bool + predicate ObjectPredicate +} + +// AllObjectsPredicate always returns true, meaning it will collect all objects found. +func AllObjectsPredicate(schemaName, objectName, objectType string) bool { + return true } -func NewObjectCollector() *ObjectCollector { +// TablesOnlyPredicate returns true only if the object type is "table". +func TablesOnlyPredicate(schemaName, objectName, objectType string) bool { + return objectType == constants.TABLE +} + +func NewObjectCollector(predicate ObjectPredicate) *ObjectCollector { + if predicate == nil { + predicate = AllObjectsPredicate + } + return &ObjectCollector{ objectSet: make(map[string]bool), + predicate: predicate, } } @@ -56,7 +78,10 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { relName := GetStringField(msg, "relname") objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName) log.Debugf("[RangeVar] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) - c.addObject(objectName) + // it will be either table or view, considering objectType=table for both + if c.predicate(schemaName, relName, constants.TABLE) { + c.addObject(objectName) + } // Extract target table names from DML statements case PG_QUERY_INSERTSTMT_NODE, PG_QUERY_UPDATESTMT_NODE, PG_QUERY_DELETESTMT_NODE: @@ -66,7 +91,9 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { relName := GetStringField(relationMsg, "relname") objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName) log.Debugf("[IUD] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) - c.addObject(objectName) + if c.predicate(schemaName, relName, "table") { + c.addObject(objectName) + } } // Extract function names @@ -78,7 +105,9 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { objectName := lo.Ternary(schemaName != "", schemaName+"."+functionName, functionName) log.Debugf("[Funccall] fetched schemaname=%s objectname=%s field\n", schemaName, objectName) - c.addObject(objectName) + if c.predicate(schemaName, functionName, constants.FUNCTION) { + c.addObject(objectName) + } // Add more cases as needed for other message types } diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 8f16517988..3ed84f4ab5 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -219,7 +219,7 @@ func GetSchemaUsed(query string) ([]string, error) { msg := GetProtoMessageFromParseTree(parseTree) visited := make(map[protoreflect.Message]bool) - objectCollector := NewObjectCollector() + objectCollector := NewObjectCollector(TablesOnlyPredicate) err = TraverseParseTree(msg, visited, func(msg protoreflect.Message) error { objectCollector.Collect(msg) return nil diff --git a/yb-voyager/src/query/queryparser/traversal_test.go b/yb-voyager/src/query/queryparser/traversal_test.go index 09f1dfd316..9e0f73d61e 100644 --- a/yb-voyager/src/query/queryparser/traversal_test.go +++ b/yb-voyager/src/query/queryparser/traversal_test.go @@ -22,6 +22,7 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" ) +// Test ObjectCollector with default predicate(AllObjectsPredicate) func TestObjectCollector(t *testing.T) { tests := []struct { Sql string @@ -67,7 +68,7 @@ func TestObjectCollector(t *testing.T) { Sql: `CREATE VIEW analytics.top_customers AS SELECT user_id, COUNT(*) as order_count FROM public.orders GROUP BY user_id HAVING COUNT(*) > 10;`, ExpectedObjects: []string{"analytics.top_customers", "public.orders", "count"}, - ExpectedSchemas: []string{"analytics", "public", ""}, + ExpectedSchemas: []string{"analytics", "public", ""}, // "" -> unknown schemaName }, { Sql: `WITH user_orders AS ( @@ -115,7 +116,7 @@ func TestObjectCollector(t *testing.T) { parseResult, err := Parse(tc.Sql) assert.NoError(t, err) - objectsCollector := NewObjectCollector() + objectsCollector := NewObjectCollector(nil) processor := func(msg protoreflect.Message) error { objectsCollector.Collect(msg) return nil @@ -136,6 +137,7 @@ func TestObjectCollector(t *testing.T) { } } +// Test ObjectCollector with default predicate(AllObjectsPredicate) // test focussed on collecting function names from DMLs func TestObjectCollector2(t *testing.T) { tests := []struct { @@ -216,7 +218,7 @@ func TestObjectCollector2(t *testing.T) { parseResult, err := Parse(tc.Sql) assert.NoError(t, err) - objectsCollector := NewObjectCollector() + objectsCollector := NewObjectCollector(nil) processor := func(msg protoreflect.Message) error { objectsCollector.Collect(msg) return nil @@ -237,9 +239,127 @@ func TestObjectCollector2(t *testing.T) { } } -/* +// Test ObjectCollector with TablesOnlyPredicate +func TestTableObjectCollector(t *testing.T) { + tests := []struct { + Sql string + ExpectedObjects []string + ExpectedSchemas []string + }{ + { + Sql: `SELECT * from public.employees`, + ExpectedObjects: []string{"public.employees"}, + ExpectedSchemas: []string{"public"}, + }, + { + Sql: `SELECT * from employees`, + ExpectedObjects: []string{"employees"}, + ExpectedSchemas: []string{""}, // empty schemaname indicates unqualified objectname + }, + { + Sql: `SELECT * from s1.employees`, + ExpectedObjects: []string{"s1.employees"}, + ExpectedSchemas: []string{"s1"}, + }, + { + Sql: `SELECT * from s2.employees where salary > (Select salary from s3.employees)`, + ExpectedObjects: []string{"s3.employees", "s2.employees"}, + ExpectedSchemas: []string{"s3", "s2"}, + }, + { + Sql: `SELECT c.name, SUM(o.amount) AS total_spent FROM sales.customers c JOIN finance.orders o ON c.id = o.customer_id GROUP BY c.name HAVING SUM(o.amount) > 1000`, + ExpectedObjects: []string{"sales.customers", "finance.orders"}, + ExpectedSchemas: []string{"sales", "finance"}, + }, + { + Sql: `SELECT name FROM hr.employees WHERE department_id IN (SELECT id FROM public.departments WHERE location_id IN (SELECT id FROM eng.locations WHERE country = 'USA'))`, + ExpectedObjects: []string{"hr.employees", "public.departments", "eng.locations"}, + ExpectedSchemas: []string{"hr", "public", "eng"}, + }, + { + Sql: `SELECT name FROM sales.customers UNION SELECT name FROM marketing.customers;`, + ExpectedObjects: []string{"sales.customers", "marketing.customers"}, + ExpectedSchemas: []string{"sales", "marketing"}, + }, + { + Sql: `CREATE VIEW analytics.top_customers AS + SELECT user_id, COUNT(*) as order_count FROM public.orders GROUP BY user_id HAVING COUNT(*) > 10;`, + ExpectedObjects: []string{"analytics.top_customers", "public.orders"}, + ExpectedSchemas: []string{"analytics", "public"}, + }, + { + Sql: `WITH user_orders AS ( + SELECT u.id, o.id as order_id + FROM users u + JOIN orders o ON u.id = o.user_id + WHERE o.amount > 100 + ), order_items AS ( + SELECT o.order_id, i.product_id + FROM order_items i + JOIN user_orders o ON i.order_id = o.order_id + ) + SELECT p.name, COUNT(oi.product_id) FROM products p + JOIN order_items oi ON p.id = oi.product_id GROUP BY p.name;`, + ExpectedObjects: []string{"users", "orders", "order_items", "user_orders", "products"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `UPDATE finance.accounts + SET balance = balance + 1000 + WHERE account_id IN ( + SELECT account_id FROM public.users WHERE active = true + );`, + ExpectedObjects: []string{"finance.accounts", "public.users"}, + ExpectedSchemas: []string{"finance", "public"}, + }, + { + Sql: `SELECT classid, objid, refobjid FROM pg_depend WHERE refclassid = $1::regclass AND deptype = $2 ORDER BY 3`, + ExpectedObjects: []string{"pg_depend"}, + ExpectedSchemas: []string{""}, + }, + { + Sql: `SELECT pg_advisory_unlock_shared(100);`, + ExpectedObjects: []string{}, // empty slice represents no object collected + ExpectedSchemas: []string{}, + }, + { + Sql: `SELECT xpath_exists('/employee/name', 'John'::xml)`, + ExpectedObjects: []string{}, + ExpectedSchemas: []string{}, + }, + { + Sql: `SELECT t.id, + xpath('/root/node', xmlparse(document t.xml_column)) AS extracted_nodes, + s2.some_function() + FROM s1.some_table t + WHERE t.id IN (SELECT id FROM s2.some_function()) + AND pg_advisory_lock(t.id);`, + ExpectedObjects: []string{"s1.some_table"}, + ExpectedSchemas: []string{"s1"}, + }, + } + for _, tc := range tests { + parseResult, err := Parse(tc.Sql) + assert.NoError(t, err) - COPY (SELECT ctid, xmin, id, data FROM schema_name.my_table) - TO '/path/to/file_with_system_columns.csv' WITH CSV; -*/ + objectsCollector := NewObjectCollector(TablesOnlyPredicate) + processor := func(msg protoreflect.Message) error { + objectsCollector.Collect(msg) + return nil + } + + visited := make(map[protoreflect.Message]bool) + parseTreeMsg := GetProtoMessageFromParseTree(parseResult) + err = TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + + collectedObjects := objectsCollector.GetObjects() + collectedSchemas := objectsCollector.GetSchemaList() + + assert.ElementsMatch(t, tc.ExpectedObjects, collectedObjects, + "Objects list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedObjects, len(tc.ExpectedObjects), collectedObjects, len(collectedObjects)) + assert.ElementsMatch(t, tc.ExpectedSchemas, collectedSchemas, + "Schema list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedSchemas, len(tc.ExpectedSchemas), collectedSchemas, len(collectedSchemas)) + } +} diff --git a/yb-voyager/src/tgtdb/attr_name_registry_test.go b/yb-voyager/src/tgtdb/attr_name_registry_test.go index a2a4edbf6e..8b656336fe 100644 --- a/yb-voyager/src/tgtdb/attr_name_registry_test.go +++ b/yb-voyager/src/tgtdb/attr_name_registry_test.go @@ -3,12 +3,13 @@ package tgtdb import ( "testing" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) func TestAttributeNameRegistry_QuoteAttributeName_POSTGRES(t *testing.T) { reg := NewAttributeNameRegistry(nil, &TargetConf{TargetDBType: POSTGRESQL}) - o := sqlname.NewObjectName(sqlname.POSTGRESQL, "public", "public", "test_table") + o := sqlname.NewObjectName(constants.POSTGRESQL, "public", "public", "test_table") tableNameTup := sqlname.NameTuple{SourceName: o, TargetName: o, CurrentName: o} // Mocking GetListOfTableAttributes to return a list of attributes @@ -81,7 +82,7 @@ func TestAttributeNameRegistry_QuoteAttributeName_POSTGRES(t *testing.T) { } func TestAttributeNameRegistry_QuoteAttributeName_ORACLE(t *testing.T) { reg := NewAttributeNameRegistry(nil, &TargetConf{TargetDBType: ORACLE}) - o := sqlname.NewObjectName(sqlname.ORACLE, "TEST", "TEST", "test_table") + o := sqlname.NewObjectName(constants.ORACLE, "TEST", "TEST", "test_table") tableNameTup := sqlname.NameTuple{SourceName: o, TargetName: o, CurrentName: o} // Mocking GetListOfTableAttributes to return a list of attributes diff --git a/yb-voyager/src/tgtdb/oracle.go b/yb-voyager/src/tgtdb/oracle.go index 759bd7cbc0..ce069d8407 100644 --- a/yb-voyager/src/tgtdb/oracle.go +++ b/yb-voyager/src/tgtdb/oracle.go @@ -33,6 +33,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/sqlldr" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" @@ -717,7 +718,7 @@ func (tdb *TargetOracleDB) ClearMigrationState(migrationUUID uuid.UUID, exportDi tables := []sqlname.NameTuple{} for _, tableName := range tableNames { parts := strings.Split(tableName, ".") - objName := sqlname.NewObjectName(sqlname.ORACLE, "", parts[0], strings.ToUpper(parts[1])) + objName := sqlname.NewObjectName(constants.ORACLE, "", parts[0], strings.ToUpper(parts[1])) nt := sqlname.NameTuple{ CurrentName: objName, SourceName: objName, diff --git a/yb-voyager/src/tgtdb/postgres.go b/yb-voyager/src/tgtdb/postgres.go index b9289aac4f..900252e6db 100644 --- a/yb-voyager/src/tgtdb/postgres.go +++ b/yb-voyager/src/tgtdb/postgres.go @@ -35,6 +35,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/namereg" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" @@ -792,7 +793,7 @@ func (pg *TargetPostgreSQL) ClearMigrationState(migrationUUID uuid.UUID, exportD tables := []sqlname.NameTuple{} for _, tableName := range tableNames { parts := strings.Split(tableName, ".") - objName := sqlname.NewObjectName(sqlname.POSTGRESQL, "", parts[0], parts[1]) + objName := sqlname.NewObjectName(constants.POSTGRESQL, "", parts[0], parts[1]) nt := sqlname.NameTuple{ CurrentName: objName, SourceName: objName, diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 5d20b0ebb0..d4f701b4bc 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -40,6 +40,7 @@ import ( "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/namereg" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" @@ -1322,7 +1323,7 @@ func (yb *TargetYugabyteDB) ClearMigrationState(migrationUUID uuid.UUID, exportD tables := []sqlname.NameTuple{} for _, tableName := range tableNames { parts := strings.Split(tableName, ".") - objName := sqlname.NewObjectName(sqlname.YUGABYTEDB, "", parts[0], parts[1]) + objName := sqlname.NewObjectName(constants.YUGABYTEDB, "", parts[0], parts[1]) nt := sqlname.NameTuple{ CurrentName: objName, SourceName: objName, diff --git a/yb-voyager/src/utils/sqlname/nametuple.go b/yb-voyager/src/utils/sqlname/nametuple.go index cdfdbc80cd..3a9a3fa1b2 100644 --- a/yb-voyager/src/utils/sqlname/nametuple.go +++ b/yb-voyager/src/utils/sqlname/nametuple.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/samber/lo" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" ) //================================================ @@ -185,7 +186,8 @@ func (t NameTuple) Key() string { // ================================================ func quote2(dbType, name string) string { switch dbType { - case POSTGRESQL, YUGABYTEDB, ORACLE, MYSQL: + case constants.POSTGRESQL, constants.YUGABYTEDB, + constants.ORACLE, constants.MYSQL: return `"` + name + `"` default: panic("unknown source db type " + dbType) @@ -194,15 +196,15 @@ func quote2(dbType, name string) string { func minQuote2(objectName, sourceDBType string) string { switch sourceDBType { - case YUGABYTEDB, POSTGRESQL: + case constants.YUGABYTEDB, constants.POSTGRESQL: if IsAllLowercase(objectName) && !IsReservedKeywordPG(objectName) { return objectName } else { return `"` + objectName + `"` } - case MYSQL: + case constants.MYSQL: return `"` + objectName + `"` - case ORACLE: + case constants.ORACLE: if IsAllUppercase(objectName) && !IsReservedKeywordOracle(objectName) { return objectName } else { diff --git a/yb-voyager/src/utils/sqlname/nametuple_test.go b/yb-voyager/src/utils/sqlname/nametuple_test.go index d6c207c496..5e7dc33ebf 100644 --- a/yb-voyager/src/utils/sqlname/nametuple_test.go +++ b/yb-voyager/src/utils/sqlname/nametuple_test.go @@ -19,17 +19,18 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" ) func TestNameTupleEquals(t *testing.T) { assert := assert.New(t) - o1 := NewObjectName(POSTGRESQL, "public", "public", "table1") - o2 := NewObjectName(POSTGRESQL, "public", "public", "table1") + o1 := NewObjectName(constants.POSTGRESQL, "public", "public", "table1") + o2 := NewObjectName(constants.POSTGRESQL, "public", "public", "table1") nameTuple1 := NameTuple{CurrentName: o1, SourceName: o1, TargetName: o1} nameTuple2 := NameTuple{CurrentName: o2, SourceName: o2, TargetName: o2} assert.True(nameTuple1.Equals(nameTuple2)) - o3 := NewObjectName(POSTGRESQL, "public", "public", "table2") + o3 := NewObjectName(constants.POSTGRESQL, "public", "public", "table2") nameTuple3 := NameTuple{CurrentName: o3, SourceName: o3, TargetName: o3} assert.False(nameTuple1.Equals(nameTuple3)) } @@ -39,7 +40,7 @@ func TestPGDefaultSchemaCaseInsensitiveTableName(t *testing.T) { // Test NewTableName() with PostgreSQL and default schema "public" and // a table name belonging to default schema. - tableName := NewObjectName(POSTGRESQL, "public", "public", "table1") + tableName := NewObjectName(constants.POSTGRESQL, "public", "public", "table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "public", @@ -68,7 +69,7 @@ func TestPGNonDefaultSchemaCaseInsensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with PostgreSQL and default schema "public" and // a table name belonging to a non-default schema. - tableName := NewObjectName(POSTGRESQL, "public", "schema1", "table1") + tableName := NewObjectName(constants.POSTGRESQL, "public", "schema1", "table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "schema1", @@ -97,7 +98,7 @@ func TestPGDefaultSchemaCaseSensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with PostgreSQL and default schema "public" and // a case-sensitive name with mixed cases. - tableName := NewObjectName(POSTGRESQL, "public", "public", "Table1") + tableName := NewObjectName(constants.POSTGRESQL, "public", "public", "Table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "public", @@ -126,7 +127,7 @@ func TestPGNonDefaultSchemaCaseSensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with PostgreSQL and default schema "public" and // a case-sensitive name with mixed cases. - tableName := NewObjectName(POSTGRESQL, "public", "schema1", "Table1") + tableName := NewObjectName(constants.POSTGRESQL, "public", "schema1", "Table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "schema1", @@ -155,7 +156,7 @@ func TestPGNonDefaultSchemaTableNameWithSpecialChars(t *testing.T) { assert := assert.New(t) // Test NewTableName() with PostgreSQL and default schema "public" and // a case-sensitive name with mixed cases. - tableName := NewObjectName(POSTGRESQL, "public", "schema1", "table$1") + tableName := NewObjectName(constants.POSTGRESQL, "public", "schema1", "table$1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "schema1", @@ -186,7 +187,7 @@ func TestOracleDefaultSchemaCaseInsensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with Oracle and default schema "SAKILA" and // a table name belonging to default schema. - tableName := NewObjectName(ORACLE, "SAKILA", "SAKILA", "TABLE1") + tableName := NewObjectName(constants.ORACLE, "SAKILA", "SAKILA", "TABLE1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "SAKILA", @@ -215,7 +216,7 @@ func TestOracleNonDefaultSchemaCaseInsensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with Oracle and default schema "SAKILA" and // a table name belonging to a non-default schema. - tableName := NewObjectName(ORACLE, "SAKILA", "SCHEMA1", "TABLE1") + tableName := NewObjectName(constants.ORACLE, "SAKILA", "SCHEMA1", "TABLE1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "SCHEMA1", @@ -244,7 +245,7 @@ func TestOracleDefaultSchemaCaseSensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with Oracle and default schema "SAKILA" and // a case-sensitive name with mixed cases. - tableName := NewObjectName(ORACLE, "SAKILA", "SAKILA", "Table1") + tableName := NewObjectName(constants.ORACLE, "SAKILA", "SAKILA", "Table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "SAKILA", @@ -273,7 +274,7 @@ func TestOracleNonDefaultSchemaCaseSensitiveTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with Oracle and default schema "SAKILA" and // a case-sensitive name with mixed cases. - tableName := NewObjectName(ORACLE, "SAKILA", "SCHEMA1", "Table1") + tableName := NewObjectName(constants.ORACLE, "SAKILA", "SCHEMA1", "Table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "SCHEMA1", @@ -304,7 +305,7 @@ func TestMySQLDefaultSchemaCaseSensitiveLowerCaseTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with MySQL and default schema "sakila" and // a table name belonging to default schema. - tableName := NewObjectName(MYSQL, "sakila", "sakila", "table1") + tableName := NewObjectName(constants.MYSQL, "sakila", "sakila", "table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "sakila", @@ -333,7 +334,7 @@ func TestMySQLNonDefaultSchemaCaseSensitiveLowerCaseTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with MySQL and default schema "sakila" and // a table name belonging to a non-default schema. - tableName := NewObjectName(MYSQL, "sakila", "schema1", "table1") + tableName := NewObjectName(constants.MYSQL, "sakila", "schema1", "table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "schema1", @@ -362,7 +363,7 @@ func TestMySQLDefaultSchemaCaseSensitiveMixedCaseTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with MySQL and default schema "sakila" and // a case-sensitive name with mixed cases. - tableName := NewObjectName(MYSQL, "sakila", "sakila", "Table1") + tableName := NewObjectName(constants.MYSQL, "sakila", "sakila", "Table1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "sakila", @@ -391,7 +392,7 @@ func TestMySQLNonDefaultSchemaCaseSensitiveUpperCaseTableName(t *testing.T) { assert := assert.New(t) // Test NewTableName() with MySQL and default schema "sakila" and // a case-sensitive name with all upper case letters. - tableName := NewObjectName(MYSQL, "sakila", "schema1", "TABLE1") + tableName := NewObjectName(constants.MYSQL, "sakila", "schema1", "TABLE1") assert.NotNil(tableName) expectedTableName := &ObjectName{ SchemaName: "schema1", diff --git a/yb-voyager/src/utils/sqlname/sqlname.go b/yb-voyager/src/utils/sqlname/sqlname.go index 95d8911f44..36bb938eda 100644 --- a/yb-voyager/src/utils/sqlname/sqlname.go +++ b/yb-voyager/src/utils/sqlname/sqlname.go @@ -20,16 +20,10 @@ import ( "strings" "unicode" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "golang.org/x/exp/slices" ) -const ( - YUGABYTEDB = "yugabytedb" - POSTGRESQL = "postgresql" - ORACLE = "oracle" - MYSQL = "mysql" -) - var ( SourceDBType string PreserveCase bool @@ -113,9 +107,9 @@ func NewTargetName(schemaName, objectName string) *TargetName { } return &TargetName{ ObjectName: Identifier{ - Quoted: quote(objectName, YUGABYTEDB), - Unquoted: unquote(objectName, YUGABYTEDB), - MinQuoted: minQuote(objectName, YUGABYTEDB), + Quoted: quote(objectName, constants.YUGABYTEDB), + Unquoted: unquote(objectName, constants.YUGABYTEDB), + MinQuoted: minQuote(objectName, constants.YUGABYTEDB), }, SchemaName: Identifier{ Quoted: `"` + schemaName + `"`, @@ -123,9 +117,9 @@ func NewTargetName(schemaName, objectName string) *TargetName { MinQuoted: schemaName, }, Qualified: Identifier{ - Quoted: schemaName + "." + quote(objectName, YUGABYTEDB), - Unquoted: schemaName + "." + unquote(objectName, YUGABYTEDB), - MinQuoted: schemaName + "." + minQuote(objectName, YUGABYTEDB), + Quoted: schemaName + "." + quote(objectName, constants.YUGABYTEDB), + Unquoted: schemaName + "." + unquote(objectName, constants.YUGABYTEDB), + MinQuoted: schemaName + "." + minQuote(objectName, constants.YUGABYTEDB), }, } } @@ -160,17 +154,17 @@ func IsQuoted(s string) bool { func quote(s string, dbType string) string { if IsQuoted(s) { - if s[0] == '`' && dbType == YUGABYTEDB { + if s[0] == '`' && dbType == constants.YUGABYTEDB { return `"` + unquote(s, dbType) + `"` // `Foo` -> "Foo" } return s } switch dbType { - case POSTGRESQL, YUGABYTEDB: + case constants.POSTGRESQL, constants.YUGABYTEDB: return `"` + strings.ToLower(s) + `"` - case MYSQL: + case constants.MYSQL: return s // TODO - learn the semantics of quoting in MySQL. - case ORACLE: + case constants.ORACLE: return `"` + strings.ToUpper(s) + `"` default: panic("unknown source db type " + dbType) @@ -182,11 +176,11 @@ func unquote(s string, dbType string) string { return s[1 : len(s)-1] } switch dbType { - case POSTGRESQL, YUGABYTEDB: + case constants.POSTGRESQL, constants.YUGABYTEDB: return strings.ToLower(s) - case MYSQL: + case constants.MYSQL: return s - case ORACLE: + case constants.ORACLE: return strings.ToUpper(s) default: panic("unknown source db type") @@ -210,15 +204,15 @@ func SetDifference(a, b []*SourceName) []*SourceName { func minQuote(objectName, sourceDBType string) string { objectName = unquote(objectName, sourceDBType) switch sourceDBType { - case YUGABYTEDB, POSTGRESQL: + case constants.YUGABYTEDB, constants.POSTGRESQL: if IsAllLowercase(objectName) && !IsReservedKeywordPG(objectName) { return objectName } else { return `"` + objectName + `"` } - case MYSQL: + case constants.MYSQL: return objectName - case ORACLE: + case constants.ORACLE: if IsAllUppercase(objectName) && !IsReservedKeywordOracle(objectName) { return objectName } else { @@ -246,7 +240,7 @@ func IsAllLowercase(s string) bool { if !(unicode.IsLetter(rune(c)) || unicode.IsDigit(rune(c))) { // check for special chars return false } - if unicode.IsUpper(rune(c)) { + if unicode.IsUpper(rune(c)) { return false } } @@ -255,11 +249,11 @@ func IsAllLowercase(s string) bool { func IsCaseSensitive(s string, sourceDbType string) bool { switch sourceDbType { - case ORACLE: + case constants.ORACLE: return !IsAllUppercase(s) - case POSTGRESQL: + case constants.POSTGRESQL: return !IsAllLowercase(s) - case MYSQL: + case constants.MYSQL: return false } panic("invalid source db type") From b956e02e482af18c994863f63a4b44669dd618da Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 19 Dec 2024 16:35:02 +0530 Subject: [PATCH 066/105] Report Large Objects's lo-functions and datatype lo in assessment and analyze-schema (#2093) 1. Reporting both cases where lo is a datatype and lo functions are present in PLPGSQL, QUERY CONSTRUCTS, and DDLs. 2. Modified the `table-to-column-datatypes.psql` query to give the datatype name in case its a domain name. (lo type is a domain name of OID.) 3. Added unit tests for this `queryissue` pkg 4. Added end-to-end tests for this in assessment and analyze --- .github/workflows/go.yml | 2 +- .github/workflows/issue-tests.yml | 2 +- .../dummy-export-dir/schema/tables/table.sql | 4 +- .../tests/analyze-schema/expected_issues.json | 10 ++ migtests/tests/analyze-schema/summary.json | 6 +- .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 53 ++++-- .../pg_assessment_report.sql | 14 ++ .../unsupported_query_constructs.sql | 5 +- .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 6 + .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 5 + .../expectedAssessmentReport.json | 5 + yb-voyager/cmd/analyzeSchema.go | 4 +- yb-voyager/cmd/assessMigrationCommand.go | 7 +- .../migration_assessment_report.template | 27 +-- yb-voyager/src/query/queryissue/constants.go | 15 +- yb-voyager/src/query/queryissue/detectors.go | 16 +- .../src/query/queryissue/detectors_ddl.go | 17 ++ .../src/query/queryissue/detectors_test.go | 33 +++- yb-voyager/src/query/queryissue/helpers.go | 15 ++ yb-voyager/src/query/queryissue/issues_ddl.go | 14 ++ .../src/query/queryissue/issues_ddl_test.go | 18 +- yb-voyager/src/query/queryissue/issues_dml.go | 13 ++ .../src/query/queryissue/issues_dml_test.go | 71 ++++++++ .../queryissue/parser_issue_detector_test.go | 161 ++++++++++++++++++ .../src/query/queryparser/ddl_processor.go | 2 + .../src/query/queryparser/helpers_struct.go | 7 +- .../data/gather-assessment-metadata.tar.gz | Bin 10503 -> 10185 bytes .../postgresql/table-columns-data-types.psql | 1 + yb-voyager/src/srcdb/postgres.go | 4 +- 36 files changed, 516 insertions(+), 56 deletions(-) create mode 100644 yb-voyager/src/query/queryissue/issues_dml_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 505cc265c8..96658551e3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v -skip 'TestDDLIssuesInYBVersion' ./... + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... - name: Vet run: | diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 52b03f19e4..2c965c5427 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -47,5 +47,5 @@ jobs: - name: Test Issues Against YB Version run: | cd yb-voyager - go test -v -run '^TestDDLIssuesInYBVersion$' ./... + go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 5452b47f25..82f7411d38 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -382,4 +382,6 @@ CREATE TABLE public.locations ( name VARCHAR(255), description XML DEFAULT xmlparse(document 'Default Product100.00Electronics'), created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP -); \ No newline at end of file +); + +CREATE TABLE image (title text, raster lo); \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 899d852562..35cc21ec22 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1890,5 +1890,15 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "image", + "Reason": "Unsupported datatype - lo on column - raster", + "SqlStatement": "CREATE TABLE image (title text, raster lo);", + "Suggestion": "Large objects are not yet supported in YugabyteDB, no workaround available currently", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index fa71b0969b..969e87da89 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 50, - "InvalidCount": 42, - "ObjectNames": "public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, + "TotalCount": 51, + "InvalidCount": 43, + "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, { "ObjectType": "INDEX", "TotalCount": 43, diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 1ed87328e6..85faaee1cc 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -736,6 +736,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index 720906972f..a767030089 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -122,6 +122,11 @@ "Objects": [], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Deferrable constraints", "Objects": [], diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index c5171c5d74..931e2611e7 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -20,9 +20,9 @@ }, { "ObjectType": "EXTENSION", - "TotalCount": 3, + "TotalCount": 4, "InvalidCount": 0, - "ObjectNames": "citext, pgcrypto, pg_stat_statements" + "ObjectNames": "citext, pgcrypto, pg_stat_statements, lo" }, { "ObjectType": "TYPE", @@ -56,9 +56,9 @@ }, { "ObjectType": "FUNCTION", - "TotalCount": 10, - "InvalidCount": 3, - "ObjectNames": "public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.process_combined_tbl, public.process_order, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.process_order, schema2.total" }, + "TotalCount": 11, + "InvalidCount": 4, + "ObjectNames": "public.manage_large_object, public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.process_combined_tbl, public.process_order, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.process_order, schema2.total" }, { "ObjectType": "AGGREGATE", "TotalCount": 2, @@ -79,9 +79,9 @@ }, { "ObjectType": "TRIGGER", - "TotalCount": 3, - "InvalidCount": 3, - "ObjectNames": "audit_trigger ON public.tt, before_sales_region_insert_update ON public.sales_region, audit_trigger ON schema2.tt" + "TotalCount": 4, + "InvalidCount": 4, + "ObjectNames": "t_raster ON public.combined_tbl, audit_trigger ON public.tt, before_sales_region_insert_update ON public.sales_region, audit_trigger ON schema2.tt" }, { "ObjectType": "MVIEW", @@ -196,6 +196,12 @@ "ColumnName": "lsn", "DataType": "pg_lsn" }, + { + "SchemaName": "public", + "TableName": "combined_tbl", + "ColumnName": "raster", + "DataType": "lo" + }, { "SchemaName": "public", "TableName": "mixed_data_types_table1", @@ -574,7 +580,17 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null - } + }, + { + "FeatureName": "Large Object Functions", + "Objects": [ + { + "ObjectName": "t_raster ON public.combined_tbl", + "SqlStatement": "CREATE TRIGGER t_raster BEFORE DELETE OR UPDATE ON public.combined_tbl FOR EACH ROW EXECUTE FUNCTION public.lo_manage('raster');" + } + ], + "MinimumVersionsFixedIn": null + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -708,7 +724,7 @@ "SchemaName": "public", "ObjectName": "combined_tbl", "RowCount": 0, - "ColumnCount": 11, + "ColumnCount": 12, "Reads": 0, "Writes": 0, "ReadsPerSecond": 0, @@ -2182,9 +2198,26 @@ "Query": "SELECT *\nFROM xmltable(\n $1\n PASSING $2\n COLUMNS \n name TEXT PATH $3\n)", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Large Object Functions", + "Query": "SELECT lo_create($1)", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "Large Object Functions", + "Objects": [ + { + "ObjectType": "FUNCTION", + "ObjectName": "public.manage_large_object", + "SqlStatement": "SELECT lo_unlink(loid);" + } + ], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Referenced type declaration of variables", "Objects": [ diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index e3ef11bb0c..b08d4f1acc 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -173,6 +173,7 @@ CREATE TYPE public.address_type AS ( zip_code VARCHAR(10) ); +CREATE EXTENSION lo; --other misc types create table public.combined_tbl ( id int, @@ -185,6 +186,7 @@ create table public.combined_tbl ( bitt bit (13), bittv bit varying(15) UNIQUE, address address_type, + raster lo, arr_enum enum_kind[], PRIMARY KEY (id, arr_enum) ); @@ -346,4 +348,16 @@ BEGIN RAISE NOTICE 'Updated record with ID: %, CIDR: %, BIT: %, Date range: %', p_id, p_c, p_bitt, p_d; END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON public.combined_tbl + FOR EACH ROW EXECUTE FUNCTION lo_manage(raster); + +CREATE OR REPLACE FUNCTION public.manage_large_object(loid OID) RETURNS VOID AS $$ +BEGIN + IF loid IS NOT NULL THEN + -- Unlink the large object to free up storage + PERFORM lo_unlink(loid); + END IF; +END; $$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index 70707a1fcb..cc72510395 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -135,4 +135,7 @@ FROM COLUMNS product TEXT PATH 'product', quantity TEXT PATH 'quantity' -) AS items; \ No newline at end of file +) AS items; + + +SELECT lo_create('32142'); \ No newline at end of file diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index bb9c5d74a8..8aa82381e4 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -303,6 +303,11 @@ "Objects": [], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Setting attribute=value on column", "Objects": [], diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index f14e1b0e5a..5b804b142c 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -529,6 +529,11 @@ "Objects": [], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Constraint triggers", "Objects": [], diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index b3d67ae3c7..ebc2540296 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -164,6 +164,11 @@ "Objects": [], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Extensions", "Objects": [ diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index a8222d3a4b..65c1989f28 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -140,6 +140,11 @@ "Objects": [], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Gin indexes on multi-columns", "Objects": [], diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index d7fd60bcc6..945e5e2039 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -938,7 +938,13 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } + ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index 0c474ddcf9..d2a1b9165f 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -200,6 +200,11 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Tables with stored generated columns", "Objects": [], diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 59c7edd001..141da784d1 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -121,6 +121,11 @@ "Objects": [], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Conversion objects", "Objects": [], diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 682f4b0983..412bcdf959 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -379,6 +379,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Large Object Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 1050fff604..0c8d21cc64 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -637,6 +637,8 @@ var MigrationCaveatsIssues = []string{ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.Issue { issueType := UNSUPPORTED_FEATURES switch true { + case isPlPgSQLIssue: + issueType = UNSUPPORTED_PLPGSQL_OBEJCTS case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { //Adding the MIGRATION_CAVEATS issueType of the utils.Issue for these issueInstances in MigrationCaveatsIssues return strings.Contains(issueInstance.TypeName, i) @@ -645,8 +647,6 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil case strings.HasPrefix(issueInstance.TypeName, UNSUPPORTED_DATATYPE): //Adding the UNSUPPORTED_DATATYPES issueType of the utils.Issue for these issues whose TypeName starts with "Unsupported datatype ..." issueType = UNSUPPORTED_DATATYPES - case isPlPgSQLIssue: - issueType = UNSUPPORTED_PLPGSQL_OBEJCTS } var constraintIssues = []string{ diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index aa76d2db7e..305ec56535 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1007,9 +1007,10 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("Advisory Locks", "Advisory Locks", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("XML Functions", "XML Functions", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("System Columns", "System Columns", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, queryissue.ADVISORY_LOCKS_NAME, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, queryissue.XML_FUNCTIONS_NAME, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, schemaAnalysisReport, false, "")) return unsupportedFeatures, nil } diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 215f055efd..6681f0eaee 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -234,19 +234,6 @@

    No unsupported features were present among the ones assessed.

    {{end}} - {{if .Notes}} -
    -
    -
    -

    Notes

    -
      - {{range .Notes}} -
    • {{.}}
      • - {{end}} -
    -
    - {{end}} -

    Unsupported Query Constructs

    {{ if .UnsupportedQueryConstructs}}

    Source database queries not supported in YugabyteDB, identified by scanning system tables:

    @@ -401,6 +388,20 @@ {{end}} {{end}} + + {{if .Notes}} +
    +
    +
    +

    Notes

    +
      + {{range .Notes}} +
    • {{.}}
      • + {{end}} +
    +
    + {{end}} + diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index dbab60c240..e07367a38b 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -18,10 +18,6 @@ package queryissue // Types const ( - ADVISORY_LOCKS = "ADVISORY_LOCKS" - SYSTEM_COLUMNS = "SYSTEM_COLUMNS" - XML_FUNCTIONS = "XML_FUNCTIONS" - REFERENCED_TYPE_DECLARATION = "REFERENCED_TYPE_DECLARATION" STORED_GENERATED_COLUMNS = "STORED_GENERATED_COLUMNS" UNLOGGED_TABLE = "UNLOGGED_TABLE" @@ -52,6 +48,17 @@ const ( INDEX_ON_COMPLEX_DATATYPE = "INDEX_ON_COMPLEX_DATATYPE" FOREIGN_TABLE = "FOREIGN_TABLE" INHERITANCE = "INHERITANCE" + + LARGE_OBJECT_DATATYPE = "LARGE_OBJECT_DATATYPE" + LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" + LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" + + ADVISORY_LOCKS = "ADVISORY_LOCKS" + SYSTEM_COLUMNS = "SYSTEM_COLUMNS" + XML_FUNCTIONS = "XML_FUNCTIONS" + ADVISORY_LOCKS_NAME = "Advisory Locks" + SYSTEM_COLUMNS_NAME = "System Columns" + XML_FUNCTIONS_NAME = "XML Functions" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 680c8b7254..228496a69e 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -18,14 +18,9 @@ package queryissue import ( mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" - "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" "google.golang.org/protobuf/reflect/protoreflect" -) -const ( - ADVISORY_LOCKS_NAME = "Advisory Locks" - SYSTEM_COLUMNS_NAME = "System Columns" - XML_FUNCTIONS_NAME = "XML Functions" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) // To Add a new unsupported query construct implement this interface for all possible nodes for that construct @@ -40,6 +35,7 @@ type FuncCallDetector struct { // right now it covers Advisory Locks and XML functions advisoryLocksFuncsDetected mapset.Set[string] xmlFuncsDetected mapset.Set[string] + loFuncsDetected mapset.Set[string] } func NewFuncCallDetector(query string) *FuncCallDetector { @@ -47,6 +43,7 @@ func NewFuncCallDetector(query string) *FuncCallDetector { query: query, advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](), xmlFuncsDetected: mapset.NewThreadUnsafeSet[string](), + loFuncsDetected: mapset.NewThreadUnsafeSet[string](), } } @@ -66,6 +63,10 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) error { d.xmlFuncsDetected.Add(funcName) } + if unsupportedLargeObjectFunctions.ContainsOne(funcName) { + d.loFuncsDetected.Add(funcName) + } + return nil } @@ -77,6 +78,9 @@ func (d *FuncCallDetector) GetIssues() []QueryIssue { if d.xmlFuncsDetected.Cardinality() > 0 { issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } + if d.loFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } return issues } diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index a791e96066..e14da93bfa 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -229,6 +229,13 @@ func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, obj col.TypeName, col.ColumnName, )) + case "lo": + *issues = append(*issues, NewLODatatypeIssue( + objType, + objName, + "", + col.ColumnName, + )) default: *issues = append(*issues, NewUnsupportedDatatypesIssue( objType, @@ -537,6 +544,16 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer )) } + if unsupportedLargeObjectFunctions.ContainsOne(trigger.FuncName) { + //Can't detect trigger func name using the genericIssues's FuncCallDetector + //as trigger execute Func name is not a FuncCall node, its []pg_query.Node + issues = append(issues, NewLOFuntionsIssue( + obj.GetObjectType(), + trigger.GetObjectName(), + "", + )) + } + return issues, nil } diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 6320e9ca35..48b8d674c5 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -76,9 +76,23 @@ func TestFuncCallDetector(t *testing.T) { `SELECT pg_advisory_unlock_all();`, } - for _, sql := range advisoryLockSqls { - detector := NewFuncCallDetector(sql) + loFunctionSqls := []string{ + `UPDATE documents +SET content_oid = lo_import('/path/to/new/file.pdf') +WHERE title = 'Sample Document';`, + `INSERT INTO documents (title, content_oid) +VALUES ('Sample Document', lo_import('/path/to/your/file.pdf'));`, + `SELECT lo_export(content_oid, '/path/to/exported_design_document.pdf') +FROM documents +WHERE title = 'Design Document';`, + `SELECT lo_create('32142');`, + `SELECT lo_unlink(loid);`, + `SELECT lo_unlink((SELECT content_oid FROM documents WHERE title = 'Sample Document'));`, + `create table test_lo_default (id int, raster lo DEFAULT lo_import('3242'));`, + } + detectConstructs := func(sql string) []QueryIssue { + detector := NewFuncCallDetector(sql) parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) @@ -95,11 +109,20 @@ func TestFuncCallDetector(t *testing.T) { parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) + return detector.GetIssues() + } - issues := detector.GetIssues() + for _, sql := range advisoryLockSqls { + issues := detectConstructs(sql) + assert.Equal(t, len(issues), 1) + assert.Equal(t, issues[0].Type, ADVISORY_LOCKS, "Advisory Locks not detected in SQL: %s", sql) + } + + for _, sql := range loFunctionSqls { + issues := detectConstructs(sql) + assert.Equal(t, len(issues), 1) + assert.Equal(t, issues[0].Type, LARGE_OBJECT_FUNCTIONS, "Large Objects not detected in SQL: %s", sql) - assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) - assert.Equal(t, ADVISORY_LOCKS, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } } diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index 41e3aa982a..01bacd4b7c 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -97,3 +97,18 @@ var UnsupportedIndexDatatypes = []string{ "txid_snapshot", // array as well but no need to add it in the list as fetching this type is a different way TODO: handle better with specific types } + +var unsupportedLargeObjectFunctions = mapset.NewThreadUnsafeSet([]string{ + + //refer - https://www.postgresql.org/docs/current/lo-interfaces.html#LO-CREATE + "lo_create", "lo_creat", "lo_import", "lo_import_with_oid", + "lo_export", "lo_open", "lo_write", "lo_read", "lo_lseek", "lo_lseek64", + "lo_tell", "lo_tell64", "lo_truncate", "lo_truncate64", "lo_close", + "lo_unlink", + + //server side functions - https://www.postgresql.org/docs/current/lo-funcs.html + "lo_from_bytea", "lo_put", "lo_get", + + //functions provided by lo extension, refer - https://www.postgresql.org/docs/current/lo.html#LO-RATIONALE + "lo_manage", "lo_oid", +}...) diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index b131eb62b1..65505d9d0d 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -425,3 +425,17 @@ var percentTypeSyntax = issue.Issue{ func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatement string) QueryIssue { return newQueryIssue(percentTypeSyntax, objectType, objectName, sqlStatement, map[string]interface{}{}) } + +var loDatatypeIssue = issue.Issue{ + Type: LARGE_OBJECT_DATATYPE, + TypeName: "Unsupported datatype - lo", + Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", + DocsLink: "", //TODO +} + +func NewLODatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { + issue := loDatatypeIssue + issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} \ No newline at end of file diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 887b2759a1..d4d6fac326 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -24,6 +24,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -203,11 +204,23 @@ func testStorageParameterIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "unrecognized parameter", storageParameterIssue) } +func testLoDatatypeIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE image (title text, raster lo);`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", loDatatypeIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") if ybVersion == "" { - panic("YB_VERSION env variable is not set. Set YB_VERSIONS=2024.1.3.0-b105 for example") + panic("YB_VERSION env variable is not set. Set YB_VERSION=2024.1.3.0-b105 for example") } ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] @@ -253,4 +266,7 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "storage parameter", ybVersion), testStorageParameterIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "lo datatype", ybVersion), testLoDatatypeIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index c1f21b9f96..3528ead45d 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -56,3 +56,16 @@ var xmlFunctionsIssue = issue.Issue{ func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { return newQueryIssue(xmlFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } + +var loFunctionsIssue = issue.Issue{ + Type: LARGE_OBJECT_FUNCTIONS, + TypeName: LARGE_OBJECT_FUNCTIONS_NAME, + TypeDescription: "Large Objects functions are not supported in YugabyteDB", + Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", + DocsLink: "", //TODO +} + +func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go new file mode 100644 index 0000000000..8f7c6610da --- /dev/null +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -0,0 +1,71 @@ +/* +Copyright (c) YugabyteDB, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryissue + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" +) + +func testLOFunctionsIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE EXTENSION lo; + SELECT lo_create('2342');`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "Transaction for catalog table write operation 'pg_largeobject_metadata' not found", loDatatypeIssue) +} + +func TestDMLIssuesInYBVersion(t *testing.T) { + var err error + ybVersion := os.Getenv("YB_VERSION") + if ybVersion == "" { + panic("YB_VERSION env variable is not set. Set YB_VERSION=2024.1.3.0-b105 for example") + } + + ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] + testYbVersion, err = ybversion.NewYBVersion(ybVersionWithoutBuild) + fatalIfError(t, err) + + testYugabytedbConnStr = os.Getenv("YB_CONN_STR") + if testYugabytedbConnStr == "" { + // spawn yugabytedb container + var err error + ctx := context.Background() + testYugabytedbContainer, err = yugabytedb.Run( + ctx, + "yugabytedb/yugabyte:"+ybVersion, + ) + assert.NoError(t, err) + defer testYugabytedbContainer.Terminate(context.Background()) + } + + // run tests + success := t.Run(fmt.Sprintf("%s-%s", "lo functions", ybVersion), testLOFunctionsIssue) + assert.True(t, success) + +} diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index cf2a95eedb..9ba4dac4d1 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -16,6 +16,7 @@ limitations under the License. package queryissue import ( + "fmt" "slices" "testing" @@ -154,6 +155,7 @@ $$ LANGUAGE plpgsql;` ADD CONSTRAINT valid_invoice_structure CHECK (xpath_exists('/invoice/customer', data));` stmt18 = `CREATE INDEX idx_invoices on invoices (xpath('/invoice/customer/text()', data));` + stmt19 = `create table test_lo_default (id int, raster lo DEFAULT lo_import('3242'));` ) func modifyiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { @@ -270,6 +272,10 @@ func TestDDLIssues(t *testing.T) { stmt18: []QueryIssue{ NewXmlFunctionsIssue("INDEX", "idx_invoices ON invoices", stmt18), }, + stmt19: []QueryIssue{ + NewLODatatypeIssue("TABLE", "test_lo_default", stmt19, "raster"), + NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19), + }, } for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -305,6 +311,161 @@ func TestUnloggedTableIssueReportedInOlderVersion(t *testing.T) { assert.True(t, cmp.Equal(issues[0], NewUnloggedTableIssue("TABLE", "tbl_unlog", stmt))) } +func TestLargeObjectIssues(t *testing.T) { + sqls := []string{ + `CREATE OR REPLACE FUNCTION manage_large_object(loid OID) RETURNS VOID AS $$ +BEGIN + IF loid IS NOT NULL THEN + -- Unlink the large object to free up storage + PERFORM lo_unlink(loid); + END IF; +END; +$$ LANGUAGE plpgsql;`, + `CREATE OR REPLACE FUNCTION import_file_to_table(file_path TEXT, doc_title TEXT) +RETURNS VOID AS $$ +DECLARE + loid OID; +BEGIN + -- Import the file and get the large object OID + loid := lo_import(file_path); -- NOT DETECTED + + -- Insert the file metadata and OID into the table + INSERT INTO documents (title, content_oid) VALUES (doc_title, lo_import(file_path)); + + RAISE NOTICE 'File imported with OID % and linked to title %', loid, doc_title; +END; +$$ LANGUAGE plpgsql; +`, + `CREATE OR REPLACE FUNCTION export_large_object(doc_title TEXT, file_path TEXT) +RETURNS VOID AS $$ +DECLARE + loid OID; +BEGIN + -- Retrieve the OID of the large object associated with the given title + SELECT content_oid INTO loid FROM documents WHERE title = doc_title; + + -- Check if the large object exists + IF loid IS NULL THEN + RAISE EXCEPTION 'No large object found for title %', doc_title; + END IF; + + -- Export the large object to the specified file + PERFORM lo_export(loid, file_path); + + RAISE NOTICE 'Large object with OID % exported to %', loid, file_path; +END; +$$ LANGUAGE plpgsql; +`, + `CREATE OR REPLACE PROCEDURE read_large_object(doc_title TEXT) +AS $$ +DECLARE + loid OID; + fd INTEGER; + buffer BYTEA; + content TEXT; +BEGIN + -- Retrieve the OID of the large object associated with the given title + SELECT content_oid INTO loid FROM documents WHERE title = doc_title; + + -- Check if the large object exists + IF loid IS NULL THEN + RAISE EXCEPTION 'No large object found for title %', doc_title; + END IF; + + -- Open the large object for reading + fd := lo_open(loid, 262144); -- 262144 = INV_READ + + -- Read data from the large object + buffer := lo_get(fd); + content := convert_from(buffer, 'UTF8'); + + -- Close the large object + PERFORM lo_close(fd); + +END; +$$ LANGUAGE plpgsql; +`, + `CREATE OR REPLACE FUNCTION write_to_large_object(doc_title TEXT, new_data TEXT) +RETURNS VOID AS $$ +DECLARE + loid OID; + fd INTEGER; +BEGIN + -- Create the table if it doesn't already exist + EXECUTE 'CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));'; + + -- Retrieve the OID of the large object associated with the given title + SELECT content_oid INTO loid FROM documents WHERE title = doc_title; + + -- Check if the large object exists + IF loid IS NULL THEN + RAISE EXCEPTION 'No large object found for title %', doc_title; + END IF; + + -- Open the large object for writing + fd := lo_open(loid, 524288); -- 524288 = INV_WRITE + + -- Write new data to the large object + PERFORM lo_put(fd, convert_to(new_data, 'UTF8')); + + -- Close the large object + PERFORM lo_close(fd); + + RAISE NOTICE 'Data written to large object with OID %', loid; +END; +$$ LANGUAGE plpgsql; +`, +`CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON image + FOR EACH ROW EXECUTE FUNCTION lo_manage(raster);`, + } + + expectedSQLsWithIssues := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_unlink(loid);"), + }, + sqls[1]: []QueryIssue{ + NewLOFuntionsIssue("DML_QUERY", "", "INSERT INTO documents (title, content_oid) VALUES (doc_title, lo_import(file_path));"), + }, + sqls[2]: []QueryIssue{ + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_export(loid, file_path);"), + }, + sqls[3]: []QueryIssue{ + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);"), + }, + sqls[4]: []QueryIssue{ + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_put(fd, convert_to(new_data, 'UTF8'));"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);"), + NewLODatatypeIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));", "raster"), + NewLOFuntionsIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));"), + }, + sqls[5]: []QueryIssue{ + NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5]), + }, + } + expectedSQLsWithIssues[sqls[0]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") + expectedSQLsWithIssues[sqls[1]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") + expectedSQLsWithIssues[sqls[2]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") + expectedSQLsWithIssues[sqls[3]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") + expectedSQLsWithIssues[sqls[4]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") + + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedSQLsWithIssues { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + fmt.Printf("%v", issues) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} // currently, both FuncCallDetector and XmlExprDetector can detect XMLFunctionsIssue // statement below has both XML functions and XML expressions. // but we want to only return one XMLFunctionsIssue from parserIssueDetector.getDMLIssues diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 43b82db893..22f60fd48d 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -731,6 +731,7 @@ func (triggerProcessor *TriggerProcessor) Process(parseTree *pg_query.ParseResul Events: triggerNode.CreateTrigStmt.Events, ForEachRow: triggerNode.CreateTrigStmt.Row, } + _, trigger.FuncName = getFunctionObjectName(triggerNode.CreateTrigStmt.Funcname) return trigger, nil } @@ -744,6 +745,7 @@ type Trigger struct { ForEachRow bool Timing int32 Events int32 + FuncName string //Unqualified function name } func (t *Trigger) GetObjectName() string { diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 492796392b..67b4294af0 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -63,7 +63,8 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string objectType = "PROCEDURE" } funcNameList := stmt.GetFuncname() - return objectType, getFunctionObjectName(funcNameList) + funcSchemaName, funcName := getFunctionObjectName(funcNameList) + return objectType, lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) case isViewStmt: viewName := viewNode.ViewStmt.View return "VIEW", getObjectNameFromRangeVar(viewName) @@ -97,7 +98,7 @@ func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string { return lo.Ternary(schema != "", fmt.Sprintf("%s.%s", schema, name), name) } -func getFunctionObjectName(funcNameList []*pg_query.Node) string { +func getFunctionObjectName(funcNameList []*pg_query.Node) (string, string) { funcName := "" funcSchemaName := "" if len(funcNameList) > 0 { @@ -106,7 +107,7 @@ func getFunctionObjectName(funcNameList []*pg_query.Node) string { if len(funcNameList) >= 2 { // Names list will have all the parts of qualified func name funcSchemaName = funcNameList[len(funcNameList)-2].GetString_().Sval // // func name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index } - return lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) + return funcSchemaName, funcName } func getTypeNameAndSchema(typeNames []*pg_query.Node) (string, string) { diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz b/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz index 6d9aa6a7dc8391db640664ce2a6c37fca31c2d45..1d00bd0b12f2c1ea222ec8d27b4eb0fa183d7cbc 100644 GIT binary patch literal 10185 zcmV;)CpOq0iwFRS%425$1MNL&ciPCZ_v_IR?^1V)n(OMWyUq`H52xVN`|bg)+;mEFog_3$6$;NBou@hn|W za{v_EF>W-|y)~Ck>tEfi999miyRiP%qy3}p`ai_8e*Hb|e6ANK z)_l1z-6G}`y&GG1OF$Ux`O;nA2h7Le{(kEER}T-4!t{TyvbzWCU-|mO|J$$sKphgb z`LaiP$Hy6bIabxy^G5R*KjEd?Xtld9NUuv@Pa6GSMyI4Xcm*k>rFPmU?V;KyJLK0< zZ>aj!P8%=OLHl1SsnFM9zkM<|?e#jM9jJ5C8x8&Puhf3~_ziu{6pKXtV0gqb$>7%x z(H+NfTw*z-H{dUs*Duws3U#o6+VA)JME$Kj#J3XsYxY`dP^34IN*p)Zol#$WtOfcLAFpIxo4P|9gHWSVzSO**DtF^22HoKs zb{S-yLymSn=A$AW&H0~RyBjSs*IgVCVt@dxjtK(HZYu;5&kPc1dmZ(77;oLy98Yv% zd*L2zFTDfXi?$Ez59=JA!a7lYs49MPtfg*mNILCbRHC%{xyKt7Z2TH6*z7er>Y%CS z0V*7j;Te_*m_#RjM^R5shi{YuZKB;Jd1dfsFjP;J9isHrV}QEbRFy)e@RTDP-4uwu znS0;=>*>E?PW2B(&nhyMUud=sBiWq#uciOH`v+0^ABr7r>HkAKTl)Wb^gm@+NYtN} z7&CIc!{209AaW$CmCgW+fB}Tsl?nts1f{~H-{9oSZz=>xte9vjLRKk|A(Bn|0&;C6 zkX<94>?18~^-db?F8(?mb(_O>uUmZH7^p20WVE4LkqmpM_|on5 zPa2(aUwxHON6IWDg5eW&1j8>8tOc-B)Abw;kR+H4@z6=Zxd;lmp#pSPNa9@h+(l+* zMwFY$B_&Ach>sd3=}~q~w=?lNq#L+cRpHgTSI6U|Vuje z6L?ybzjU!y&^of2VdHs6Rmd=u8Dl9c>cTSihizn|Nv)9ETcV5={9~2e<@x7yFc>}G zv=7$~l&L%QuXYA%yw&cj&Tc+hZzrum$dCjF6?h=5jhrv9owUY6ItWQjtN@_&3Pl(- zaR9*|u?yqJEob=>70gFSpaYn~gOrLeVUCsvkBb$OJG#4uZY68zQDD}o=vKO{7)Bl{ z?=alni6iPxB5H*to%>PHv9(bbS|^Q8Cvh9E+{9stJrS2A2^n!6fN80}DN^xt921gE z!aS!MLN+vmtTuvt!HnQ&R!@l1?!s7ri)rmGTEn0XP*~5TyMfr7TG^Yp*kdh6qgROeXWrBtn3L z5B57_PCE4CmEj+%O|3lu)-uHFiC~;?8upE zT!o7soXRVc(m|4%7->0CTZ~UwJpt_!fO1y=l`NTj?m=!K%p}M$8y()VLZ0r`3UMuuNL@R7wSBu-Bsp8ns}|A%7#J==c|5BHgqSK^F(2=I{DBMVLuxoNG~WN)#^cQ;Iw z;lBo6-Bo2-hj-z#p$A31M3H*c$;(O!`EQPZF}&DG*I-xTrEoi0bFNa_;ABNfJWBao ztbz`GtTK|Z%6EmCe!yi852cRD(rSl>z%qk`gyoL57I(JtTlikwts>(U&wY4Wt@NMr zOMeyLC@d7)Hlfwq2{rJwZSI@uf6U*}HU52_1A90U%C0Dy2uy%He; zhBH3D@pN~EV(EUCn-Nd|eTK9m(a$KzP4)vysU7n)3b!Lml4&OCwOu6m8m0HLN|sGE zimz}BiU5@+ceo|GXmm+Jt0VK7iOe7@g!f3Kn8Gh<0-n7j!bwOmeQ1#d7W$_FVTJu_@_t z5daFs`H~Xxsc}J_K0Jx=k?B%L`bgmk)`GYeiraIII*UMO3r3YKbG4gOn$#U^5GHi{ z7@sa|ClIi|Y)r#nU&t1#wN|YD){@w_dDhGSH|NDG>qeVFDV9=9uTyd_6Z%*k|Bu3c zH5~uHw+Fwr^8Z6TkN#OcH_S3frg2p`o)NWh`FMLn+jVb&jDzM1vak4UOdqN)pZEVB>S? zd3LQ_zP`RLX%ys=<;==+R@5zb+D)}PP>TRIuQW12IB|)izh4>-3r5mxfNFx$2bpWv zsF!FnN5?~C3!%Pt3}jv)gY2=-*>9f?$E|k1p39TT6eWNx{F)jLrM&d@BLMhvJm`)3O|@3cRX$}& zAxnPvnUtscRoPt5=j4YUhcH~{4C#6*%F{^RwwyJtJPNr1N?c~?+trRV4-y~Y@QbKFMER_ z{JykY533gPr@cNVAOM9zzS-+`LF5>>8p8%w2c`TK>^8QvAa&`^6`&to!9t;+tO0WO zfq-nc5H@4F0Z4l?NS^7AHK1MYx>~B~E*(z(!~%mdvaKz;}jx^-@(wJO^_JBjB^K+d_k%)>w)L|^=xc4jv+h09XLpnth#{lL)YigdSFW?2> zWRauHlH)^QToA(}Qwy{Mi{h$F`(QzB}`i*8s9iKJ^gV()&tNxP>U|z%S z{y9u}6c(s@ABa{x*sZ1_TH|>W5sfG>`|w{Gr?Bi<3kt@ToOzfhR$!35@G%xkZJDq_t+o=~X@nza;@(8==;7ZP2yUg=m^-}} zYC%J?l9gQ*D)TSBQ zH^=A5Oh6)AGMXmQ3ks>vDYZGE*VJc}%m4*>bCl8mC6_3s(@bW;zgWd$j3P^v?Q(iL;i7>pswt))Z_` zVeJ9thnEOe$JQL^Bvly11(@lk?jWXecGennGql>6la8rtMkW^1f%8MtU$EAAm z>Wufe?m@HX4m!XllX!@BI*YZfM-WEewTqigOK7tSK*8QQFQU>$+J&%zdMwu zD%rr46|6%&J3TjQZ1rk6zqtT?Bv3}Es|xJVuHT70gtMqiU@{IKpb>62A3OmaLWXS= zkCn50;VnoLA?fO4OxD6}nm15Dc}fiOQ=N2=;m40Z781(xrO4+D@*h&BlFpNQIjtl< zE@;dil!qO%L$Y-=0;Cm1B&=&^yKmEr%-rR9zWm1-`Q3Z_<1PD>D?`KC9a=r5^-pZW z29SqTkoy~fyo0`ABq5Dx(-DCfd5;4zre6X-XiXb}O=aWzgBvN<7XVIm!4$}2so|9H z8y*M72ZX~8t!>JnshDsei&k0ZUH9Bu*uYFcy@MK_%CsiHuhoVOUrdPYUIwL(1X*PeZd9YVj!@X%7F- z2Gmui(Hr&v^Ol4E65G2l1gDR2)EAsnPNhCTrwCp`0-u~56B-_Ho)iRR_bya5OlpycIT-3&R1KNl3fS(j4>Ov00?4(YZr|_t>+> zbnijt$K6^Hq>4vuGa~BK@p8F|B~qE&nsJd56nv))f2^e^U=q;K>D+8GeCdkhK8D8m z33i@NDzVKL(qXuo2skV{y3k-?Q^4MQ1i2_)&@$OVRw55_`ADJU~CJ zIlDCgR#`wE1_-Fv@1|^44$W`jcQaW*^Bb9H9;AmVt*OCvR=%eJOYz<^f9)TUz79}d zaTC}%{9bxGGmXJwrRcV_YN{%qpR82jnWdfWB@V*?l%mNcwIQZh?{HFKq4dQ$KM9yD z9eQ5xE9^09<*sYAo^B~%(`PT;%aucs4-h~MLeQ&?FAt;wa@(QnDj(QQ(+PKolz z*&pR+Z-3-;Mji`S0`O#pw+xsoesTqraiM<)lvfxHeMS@nwNjvvW2tDT70R2~0yPq# z59+R@=td2Zp_T-4vBlIsAVZ-f3zi7aeLUkzk2XMQ8y@1oRx&M*n^&+VmfSsk!AiF3 z(b2HMrkb39LWz0#RL*^jRQ~iV_c5(1RhMEj?@I;9^-}ngZvhAl6~4KUnZtdH#cJSl zdD+|mE16a@D{Y?$q2q8j60F(WP|g6?=%EWLRiNMpMmmKX-MPLG+1#KFBrO*lww0d< zC)=lFYQfMumSw}p9cwweB)SG0Z}4m96!JxV+`;+>eS+l02t+fM%Y~hQE`lgUGqObV zQ;1g4=tx#q;{053%f&J&&m^}Fpxk3hMv>IjQ};NakObjl(e5X7rp%`C8XtAobi+#K zLNJ#~rJ$~CaH9NIfLsb6hy`;Pij^c zu1`VkEZUAWfk~#IctG?5IWQ1ZsB0=c7g1b-hbK4UIC17npThYyB7k4SAV`$J<+geTZhJKK7yA|I_l(Onx)bb(+{pEt`C%;ulIOk2DX<>Vvv&Ezp&abhj! zQ#wR?7`-&7zIC4WExJgu%hRb1$vO0|vj9o{LYyLk+_89vvP9A5Nh`>ww09E@+HwEN z^Zl}SqVfsP`aA~{Yel8Jbe;0tI2YbZyVde!ZUCJ*+*dfi;T1*$wJ(-U@GzP`XuecW z8X`;D@HDs89-wRJxY;{7QM*F{2bbbSuJ6An@+&MKIvqp*$4xa-5h^q~9kny2j;8~W zyU&|#3>#y--)wx+I6ZB5Ux@ryjehfGqrY1NmN&W7oSkkg1?s9b&sgZNt_ypo)9b#d z;nlYE7wI=%*U+O!+i7-s&sjz{`1u@vvHa&K=O!kPd|+_xZlnKZoB|WiW!=ZKhwT&8icU_Y zx&7^=gNs!Au$YVLQFu-}gIJp16nWhqz9jfc{?+SBtH#}(;FbIvs86&;r`K$BHmJvt z38taBAv>iOPO9|RHX#8@Pco~(eQLMljATntNFZRb*&K) zD7jidywd6o7#+nrGC@l#-ZEU2n47?3P>989iIi?legERVPzw=inAQdi5=MJ5Ve=X# zm8m@gf(&nL{e<1R%Q{Y`m+>abA3p+bEj7?}3Tl?8$$g0Y+Z@|G&0hDo{UWhB-JC|- z$;Gz^=(4#e>WiI-gK@dD4b#zdX1k%+R@U=b92lVcK>c-uu>$auG7b3aPc;4A{H5^% zQ(v{!*I20EegR|$ehIg6EYn>*T2~XrFaykoYj`LtvfY5ayqM_*Bky0Gy_ITsDU6|EnrL-v;*4BCpR0d zj(ZsD(`?ill4bK_F%$-1kwFcD`J9DG1+YL`@UkQK666F_1s`A7ktzlvIipA8$pSPE zPMe@;@jWSe0vIdQFe#qPEs^3RjxeNHG_CO^5H#Icr;PXnk=x@!Gf|_e4+k(!0g09f z#9*ESB#C=P=TkTR<2a$%1(cMRG*`kfX*xre{7W!B(F z2#`Q$+;~P<#b7pq5wV66Cqypf^BA;73l?fQ#>~La3rOBB7ib4Ump<-uOWi3%Op9&V z(~ytI=NlfV$FvRvBr9x@Yy7`A`~IUG?PyrGTe2Z_V;}Q%_dg!&?M1%-cX)Jgu=W3c zh^Gkx^H8M%9bqwWC?~O)Jkl7*S0S1u)3rr#Su6r=QgZZpYUo+QrGK=-JHROnAacZw zxU^$sRZ&|M^<&Az^=-4x z&a|#gmK?w4cZ2z&kyH*wQZ1AM>(+LcUdh$pkG-WmMx*iuQ(?PC$(Z7&gfvqI2Ajbq z+yIojLSj-l3?Otr$_Y!LbD@N(KEDH?p|MFfGGf4m&Y`U-3tdBpSU>iY#yUn5gZmqi z#m#~Ikx9l69)vVRUDrnKz{J2oAG=vv9X11~w~?Cm@0Xv4sf zO|UWxk_$~G@#iw7Wl&`OfJc}W)-aE2Th%z7%y^r$9{f}@cXZ$?o2&}*yTi&M{UaGB$0Je%=<$=TMP-FN)o-r>=H`1{X$d%KnG{m&2b ze6#q!n+J~n`!A$H-+A;OQXafB5#d0pRTzX*2ny-EHj<-Jro> z`Ul&=F#wE>F+tQTOVWzEa>DE?lguQ^VGat_}h{J>aBeEOBz>-XQ>P z&4@&lgJLhwfkxcA8c>g0!@PnuvYZ>`FA_k9xaPWJ*lj=rr_o;UQ!hXzOdVmT6o{TN za0wRw(;0^AA3PutY#GIMNAnwTAgNDx$eQsmYd;{-l-;5v0J0cDyVlZ~1QEUzaFpW4 z6V3LZ)|es_`p2=SDVn3T{Y=CTrap*phj#CD5Uc@iB-T?Nc9N?DR8zN5BKeknp)Kbg zsqP{{lA;bzU#t%}jP0{$)L{@eTJ2zeZ@0X-5d%}g__i)^5X4$%iIom3f9oK^{nc@N zU&J4~&2T;v?o!OP4H#5>hhb?qCs4b+G%- zj9&}_7t?CpXV{`VtM%M|CUIKpKoH0yjHaXH!JX^6P4@`w@5z$ie1tV&vU(tSIe@&8 z(mrv*X#}bH0=Fhddd@{-D6hYdpj0w&xdUnTBE2O63D}rSR5>Ng(uvtdblwrv?gtG15^b>&v!-seuVee%;!KT&UA9~?Pkg$-^@y$I_UnsB*fzRXfdl~=fj+f97DKXxtcu47nhZ50Fn~>IN+<@}Gj) zO#SVVqXMcXS#yeBT8d1My?=HWbk)PElau3Q=j>TLqjP#zxu=y}fOZ~q|{!O`Ch+%-pA9ME+E0~FR zRw&{zl9yfW6$MDd`3k=+f#$$~xjhyX5d+_?1<-QCmB8Q&nA9V#;f?OXBGOwCR+UP2 z;ZOEF!5g{pEBA7A1w(8=_}wV&6=ag@dpNvv9friDCVY*dH$Rm`4eEo(`J~tn#RmEo z72l_Aj9^!YQ%bQb0xJ^+MTZY|*+$VzP$qV^OHUa6Eb4d=GhvB4A42zHtTA+HS0IAW z(*v{?(LvU5RH0#Wkyz#}I1>pX1d-Wv9GXy3x|hUiM0%|G4bif9jk!uT3`FCtl1lL@ z49Pyxda@)^_yX@JIV?+Vl=q%@3+v;u5FJC3%=!>3ta7*bDvI-|!BuHg`A&gR0YO7z zlaodPV-9WbZ1fmhy6MD+L{bCmS+-?UH|vGy4ntc)3oow;h8{h`+L1_%}*22DktIIB~NRsP7EV0OeVc!l|1^)mI zG5oH_H3k>T-&h_SB<_MZjE|TpiSr%7V!7=b+NX?==3ZOU1>Q8aKy>NugzHk~JO(~D z=3E&ki}7kHH3kZ^G9`~HJ}uu6)h^Jjb5lg4h5sCAMnR!SCRcwNzTzZ1Jukjr>W-ni zMR&PCkD7#yF7&p(ig&h*N8Q2b6o09$@vkGb{{~&<2gClTIUF#hj_Oz`I-S3yiF9o6 z&R`z?VJcPwbxO$HAnz^B;Eq!S|At>YzML>objku1rlt7&;Uh$&Sf&tQ(iYk`w}x%L z@>{$|F?*p>pF2%l2v41+L*@0jEUQR$rW?{5t1Rb2lF%(V|l4!k*D#i%9M}Hj+oo~r1-a& zm4f8zr-Z8;bpPsxNas|j4s|UV6SJ{AdL&rg-_)KM=?r%jbrkY0H4~cg50RGk%+aRw zU{w@_PGG7ojoGE{tPZff&v;sSDFQ_0qewy2B5SK?ne!X}ErO**U%9C>lRCzz6eldK z%w=!vGZFMBfPZDo^_kAIn6{337beUtZCboCw4(2NU34r9d!g|@+_#iRe_+^;5)b$Y zwgDfVga{$@j-Y?Eh;O?4ZbLgZdN%X_jEe?$i2zv>|54fBta)hVL#DW9X5ZxesrP=jIOiQ`ns__v`=HNe^q<+1JT9*n@5lny(f{f`#{V8x_p9)>cR(t8AQNume;?$z1O3Mk zOxSxe7hoUMV_R)U6p|T1@Wnlr?VZn~ZLrUH%56X)xee(%pE7M^RLls`Mn*5gUsxBE zjecMqC}X@l^yilsr4fc`>43`esi3uj0SgK&CMvx?p9ZWi4OY|_lN1iXj86X-gkd7} z4b#^&%9yh8w<7gl<+;oK4~}(RWIi7EeE-AIQRMsYN0t3;{O5x_#Ug1kM|UkLN}2{1 zvEzU1ny|&stmGMa`8b~vDh%D=ggy9vbdZP<{Ip^6y#1ox9cFkO_du6D8=1LLI33V0 z#>0zSl5>YW5+-~|l6oYU%e2&Hr@`rHs0KiB_LkJ@BuYVr$$>jn<&|h}N`aBpr-|cz zDvj6CCwTW~Uv2jKt*4pi>I=>i@MG~F`#1_lSh9Ov10TwWTPLk@+=IDqz6|vT!ZYOp zp^38{j4*$noYAoN-4zP~@@9rN;_diw8{=2__xaI#3 z@qjeUP$2LfO3(;6ontRFHj5rrr3eqaG`b)~M2Ja9J4lfuqFTYEBzK?69(}No9aDW3&F z1HeLWA=cDSKNH38SI!9cfiSa6pkZ``m17ZQ`Ujg{hC@WQ+=yP9k?h>c#;Z}g1F~4W zg3IN1a0L{G-orJuCT^KqBFb~Cbm=YTk2<|#zk#<~H9BhTKlCdF_2H%dc(<~!rpr10 z(B`fMfB2;bE~dag90L47FyYS}N;Kgwz2^u18dwa!6EMGlAO8nbKvbhZ!w-ZQ>(jmf za^6;0nv++T&7i5#AV8XevnduQ*fo;;5G4A-gXFl!ae|{iiW32+kL;xSXRzVB-j&Ec zF%;6AWmYcl`am&G;29fob*a5AZ~tn~-S2;PL9@Br0l+%`Uxmbo{$Jg%Zu$R1JR95p zS!@NM!*agv2w-bA_*Q0vu%8d~!}$obny0!VHI?Hl(A6>C?CDa@LlSDsDr{1IM4e?p zv-OBtyY)j#XTHN;t5+kJz*gt%20wAX9J0JTV}De5atr*@8^-G~Dep()Nf2}gY@hA(eLepV%Y9^w0NMZm D(|fY0 literal 10503 zcmV+iDfreOiwFP!000001MEF(ciXm-`MUZQsG?L-uBn$_=QvSr6`4-7btG4kn`T`f zU5cV*ZYYu=NISkw{`<}hKoX=RKa$jG@0#bN7Ks5cnD-1oc5Z~{R-hT2TbwT}H`EqZ zXv~bzs6YBvR|Bry-EI8a+}&=df8*;>b8Ba_)!N-@G+K`u&F$UR<|DTK%`mv;ig;)Q zj6I5`ksC(Wz}?XPH@)WT`R{u?oCg;F;C!Vh+>Zadn~lvK;QQvIMyt`>+ujvQIb!3^%R`-o*215J#?-M`XH*#JhkAEhU~r)XT-mdY^rmny9K501 zJ#L;`3uEFM3#;bOCqUGY{$rs`@xt5J`cNmx+0>q~Hmit%nFn_`3WH84byI^|jG!6Z z;&1?LMnM3>Bmk9a^;x0tyXpB?D!QS^$}1$*v_zlTfHSU0dh zPnkh#;02&EEYT+(dI9`6w}iqo5QoSCeZGU)aBD92Ns&7vrkDv~N6cDaIdQ8u{a?!r zmDX-&q%-=37np^<=EYrm0>p=Y&)Hj0A4sFl1W@4S?YkJvnFOvQ1 zAbYFEzp>XM?zI3G1h(OTa?RL-~L&Ug4E5=>o?M|jwp~cCDvlYEf2aEzU3;#Xn^;fkcm$+}37IWF4 z4{Y|xuqOxeAK2(jsx$>Qq+;EM`*)SjaM*cw!o$FJ z=abNz7(rlMRrbnK;vJiZIPRx%^%R{K=-qg*KiFgEhC2hDK}Ca}q-wT%=2cA3i54y+ zN>b!k-8Ke-GD# zBmYY}a-`R~*>YYDd>KuhfRhuH=~Z`d+#gq-R+Ajd2*E>5qcBdg@P5hV!b3M&uotY6 zI7xV9^I72ezBQ{}i&@5fVpsjQ-e^2xl?;Lszh;7%2nGaFYa#*F5hsIvpT$eQ35z%( zgP116x!@^&mJ67&ffArV5!7D`O6j z{#;w%r^e4$%yE7Fx3#&myCv(tyUkX!`KW=v9_oMhay|Z`KDFKY)ZpiZ$LM%KH3NHo z9)fA8vUkzkz$|h9^Hn7u`V+aLGz5hlakL4UtD?hX2IcYyYo{oL#C zZ!pV7H3f3|0j>cMwUM0GOwev*Ss5hGJb@JVEz>@;O<0{fk3i?y+`F&>7qe5}3Klj; z#&SS$M&mg4!VW3Zju*e`NmU4E%mPV!!i7has^rOaqk~)k5L5 zet0w)>%WcL#lkU%RqU~n0?(eKllEK}!Y-?e*kR|nNEPTTNwnnpfm&T)%B&I0BdVCAOxT#1 zhlU%{C=0MuY>Iqx9PbjjSm?yidj%17@y^k^B%!eq;3Y;tUA@V|@RK^8{X8ha&=G!hr z*C5YA;GhGKEDn@lOkv#8R7bXNY21Q<-oP;l1_&;?aFYDXbK5-6FfO3DWQqk7Vl`6y zh82A@K*4~mWj~sNjrhb3upXam_X2q21y^OnbB##R04NUd8zuv2=7XCByewIOO*^u< zBUnYC$y8L2ZpLp+q&5Q^tSj||xEA845Z7snB5;_&z&>Mk$Y!3!$xK@xfi^^Jf6n;? zv<-~lEI-S>6{3}JEySHH5DvP$Cj`TVU?Kz?fye>Q3XHd1fB}lnHrNg80Z~4)=O9D? ziW(9l_UvOv4YM*Mx-W!f@np-FP@zgp-XZ9Sh36rn-xV{k)1w6P?tDwndcrrB&NJ5o_++GY!uGybOIO( zBqsIJsTD;n29t7O&#YNI&Mo3$16NvYZ{Z539MwG9S=J!Ug$9=Ky!KIsVPw;AD< z4Z1?;3B(#eVYVLK38LwNaIp zut%p) zAOQHOs4VBCRGA@9wTS(r{N$bXWT8Eou_v$hp1j$6GAdV3*-42;K4m32M3H@BAwNiL%ab1@T^5roRd>ZTD#U?N_1u!Jc_BcpF2J0TZah>#zpz=}g1+y$7(HtLM$ zIHH9gKPwP2(p2Oq0}lmV=$T^73H2l-G#t>xSrQS7tuSbFFm?v*jW89XMlR$UWd4Me zU;y+izn7o>US_AK?9->ryF44Mu2Lo6Ewu1GD#_MD(4#1j7^!k6MFtrtHwS4!i**d=gq zZiPZA=Pm|=Gzq5Ev2>8#oaWhcnQ0JVHdvm48BJtGeK;HpL1p5J z&I}t&^IkG@G3Ax8;wh6<&B6fLgD^e_<7AF<2uw~OSI26n3_vN0lHpjHayZv?BDBPi zv_Sp=njl=ID3Rv$LG_|dr2%N7c1!d{qC^B&(5m>Ee5b17NnLvfqjvdInW0`WEy$10 zoH`P>j4n zD~Yk{v>3r=TBop-_sJ7rApV(ou2oP?f;tAyNZ2n=r(*t(b|uEE8ViSNrnAd@3d=;< zSwM77BPW45F@U1F4F6Z5$+OBm!ZH!wBpqWVk2_YvLf3 z-~e*7#Z=~O^4MJ({-nOB*FPA5TJf6(ZI{;}DGj0@@$+k!qLzNZgrl0KJZ0Lyp>LW` zB@rOY+F%VNQpdnVt%Ss>*B(o)fY1o8q-dFeVhIo=wLW6GgKRDk3hQQ}4O|90g7z5e zUZ4{YSjs37fz^TOaTplpd$EDUC!PzOMUM)ix~iI^g`E#3E6VLHE)9OXyCv_r+}x7; zGi7y#@-r#LGA_)`USvOj8sgs@%<9|Dun+8iYfxEmFS81?dK68|Y+RO?vLz@=v_&bb zm}OP57nYTRYd+*e2Jt!;S;6!Az)%rpQsqV*X`x7`PTOP_ud#hfPN5wO9+y~56UH1y zi>!aL%DZ~Mx+DQ{`vk@?BT84}_T?tvnKTB=Z4^ERtbDQ;`GD=-X&HZG{SJRuL5QSZ zq1uASs6`y;p)kwfHr{&+u;n&a(2xvVH6!xy%HqI8v;)^dEi-xrv!$-}Lk$bqR%(7)j ztojzD#EKAtscw21n+z!FQp}CqjC;vI1e?UZS$_ zjZH~e1*|!v;6?ScALjbSYNnkqQPRM6QK;-%Wj%|hW7?h(Yv8bgwPAkbe_xVv(o?%Q!w3r6lQhVizlG-!MKoLTbJAAObu) zvnUn}?L}G}=Ns9+?}D92ip z(3n^PQGTc#ggacH#E z7zZ%x7@d?`vo&A!Zh6L-dKsuMC6f%o*OF-^hDZ|3OmN$iru=%Ue6>Of*O?1QRw?Ce zRZqEN-Ob(4mu`T`(+p6@J}2WQaxBc!!TAtX_|Znk&bcs zo|XR)b;Q!<=dygH9n{jt&CliK02c%Xa!*wL+~gEgXQoz)iN&Qh+7fJT-L{^Xn+&yz z4hn_mh6}?an@EvA4~!XYtj3@)36SdCo}XJmZC(A5{x05QSw{1vkX#e@Dk0yB!zOXq zmn+S;lU1w{R85Sfd9HJn+~uTHOOl(mD5-0j>uERxr9O1$UAmCO(oaDD!gj2=C7VRH zj(Hb8a6DtCUD$?}9KF^8&%;^pU|3t8C#{P=H4xZxaY-OjHIS=Q1R*2dIq}bfB&9nn z&m;drI;&&P$*n6pf@fG(!b20mwpn4F=J%5(jEc)`~g$~16A-WY=Vf+^7|895w zQ^XRX999q zAoWEM0g%225>B1`7&KL83;or|J{$?(=hSL|LocRb*J*z zDE`C2{Qs2ecJ^O+KWTDD6LtE;o#c&if)?5)vI^29 zo3&_`R24scV&z&H|BQ}bj>f}Y|5Zf*q`StgDuJDxmKna4G_)4pg@pwku^}>{T9?>T zB`u(B4lo-Q?!KIyUW?fANFQqZ`a!R+?_ZCff)ya4eCFerTS0PkQy>uGi9{B|XC01; zfEdU%=$r~Y!9nBxDPtAals-%xlTurfXiE;Et4b4=9JAYB_LY>rcS#30Ov9D$39kHP z$(e-*k?0VfIA}xXwgGq5;;Vw0e#T^lS-Yk2SH}ChW;u8N{ugaC-R1LNH-7(Pdoy4E z*=jW!5BC2)u7~*lFBbpbf?jEO9e?|S`8*11-1;yHBY%S1@V6j^&mqR1VfoH*Rb+4& z+R4Amr%1AxxP*rd4&fi_gUFv{AIvQ04v1#fCXo%i%ZWH)Ah6vl=XdSP2$p~NXJuJ* z^b23t6B@*6v6N@wqL-7bEGbz+%QYh#jx&)b8<8qv;I$cCj+kHfX_2qVyOQMX&&?6P zAYfP(C9M9;S$w+UKdkO~7yl2sCO*vat^NNl*8g^P8@oGOn>+OV-`4Jf|G$^(*5CgG z<}<p%WkQIB(yy}(y@<;G5u*(ndvg44k4W$#t5KQ2f;9$D&}En>SlK6DCU$rx{V zO>G60&DtMS%>gTw3j2EZup@}Mt0+ksrc}pVip?L<3zIM9o|6|rm z2bioO(^tNCp*Tu~O$lU%Gck8ujxTa0Ye=9d0f4o{==94NSwd+~B0(7;cWB_ksZZ$ok zgmEE~_*o%)hrdcCVOsQTiGZC2-r^SDLcJzNP+a5RD@M4+{{taR$NC2U8<1Q6{~uaS z@cx?!*W7J2S`YvKhx@p0=Kqp_L+Ix_5x@N$H^qND&7FMw*VulD|L*0&KuYhv9>>a$LTVbN@5I?Ph?+en4^7H|-!ARa< zV%pcC4+7AkO#IB~81HKEuRGY+Q;@+(fjH>&4v!%eEC_5?nQyh|71qS*#azjYt*g>- zzs93!QaX|m+K7@_3}KePRti0oxESmI|JUA?ekYM-@$dc>)igE`K}>>r4i0mIfRWQ| z&7z%&%8)>iGYQZXXxmBVZ=ZYb+pAttB*8eg^*$s;z3slG?)H{npZx#u-cyVb=c~n} zW9iGimWX8l5x!y!XO>xnR)X%K@e29dcAC?8Q%mChr_}{&0WKM5vS?G@;q2MR5#!z{U= z=9yNNufqzKX=fqIT|<;03`>z-u%+4Tx%{mBWzUU?q#RHxcjA}bgmftxxpA;-)n6-4 zqTG%@)DulXg&dJ1tkFz3#9-)R28Ty1)mcW&Ak7Y-Ms^1oi0GnCfaTjgN~EYh@=JA& zI$#c?bE;i=1r#?*l|#Ft(&V-QZA#?f$>fomG-?B)yu^te%{{QSL z`G0axa=E9uy_BE_!!Kd-@rCip^c?Rq#MKQSa z#}9;8{vB)pQm@ZDlVm`C6eT-d5yTOS^mJLpV zxjIIA^njEuK2o86CWFkkdTDM-@FO8BR*2Pd>{d_{e5QP{6Am|_zkSUF#9j} zl<+^F?LB$EzyFNw|9u|ef80xm`2X$rfAGNw!TpW1G5SFm)9LL!-eMO36D0urXKq*m zRfFx|4NGGf072gOiO%KB^EX0dI0nlWE7B7S+nofnT|qMKLzAqqROGMCQ5zTN|+Ug_JJ3IGcE-nVwRkggTEpj=G=+QUy2p;T?c<29hm})5f^f zbn?e)T?bm~8}kDXOyg51K73vxMfYD}Fbi4!^=NtZj|F-rFf7+)7M z6}xhkT^oT19Pv_rb~7T+T;^9EWBE{*%xDw}-!M7%c|tka0VOT%M!rDdcDyO<NCFdwFGIbVZH% zHg*CAWP+Z#q?l^KkjqSMgAh(QAR1odj(20<%&tU=K;ZNdyVG4`Sh^v4o^kezGgsOS zJsQiSRLEqxY1uVnZ5yNmbL&Qir5s(84BC|NPrezZJ`kpK z%NZZ6Y+D%E-*!#w@2_Nw(F#?ozqKazZ9=&Iufg)MH`~U4fA&1LYo7nuf3m;3AL;-1 z5xyDz`_Vn)zjNO4og=`1@1X5>!)#~#XC3Y>7Pc8~JKrMkZ86(}8iX^I6SxMi9n0jH zAV8vy=?dv2hr*3}$yvq&CRpH3ngv~B7Pv;A~QT$wj zp5j5&Pk65f17D&YEc3;PeZr8zU5#7$g8@Ca38WsOg~yzm(E7c=h4ZB& zg4vi3B6M&&I=GHIJs&Kfl0an;2Vgv!vddk#I4gG=4=7uovyP!V^?`U#iH(>z(!mieeeYJI8-|a zJ2-xbbgIYv1TisPTo+}APGP)WiYk8IIk@KVuEi%g1GLYWirT^w3rnvj@?S1Ue1$;z%QtNpLld2uWRope5>J59@d_RuJxW>9@}Wq^C(eJJENMDBg=|nt3@%vAOGpo2 zNZN@NVyPE_NpTVu2FH8vgOiNi)k!k@;n;b%eE;Z*{flRz;8>cn2b6xzK3GM$VS;S0 z1^O3shn5%?(WWf|HFAp$nGEkjKUhs0l1;kt_UJ~AbqB)977U==_tt!4{=(%ze}k~u z#@Gfln$(0dCq^H{L)V;E@S9rs(B*U9oG+{qeqve2Z&)9UzO9-%X;{#e<)&5E50ZsG8={RCgRMY{U6Xkgk@vy zSf)fUQ8}49|7HSFyxIHbFS}N){NK)JP%g&oem{0%YBZ!u&=D;kz?+eeTNQjymXMhJ&3&A%V$8V_?dl-IQ6N$TyJ78 zN#~x8xk?EJ{vr$iW~-&3Nl-)GxmjmO?aGN?8tuG>FK^^L*%TL1RCG3gh7Do%COwmj z<^_Y;OT{$yz?!B8m}V@3+0v2#dU$|;%uq36qbw*l2?Fl*ZmPSLjs8u&o7xRDS;*;N z3GaaP$q@`Qk+*f4W%Gbhhm8+ zlvow?(+cBFd;)n#O&_#fos8>sL>eWD$;Aef!5a|Ef+50n9|Cjm6hOQU7jfWIn$DKo zy+XGhLdw0?Axl{f;bHVKWYL|k*yCAia_+xrmB9ooRs;%c!i zyCXK)zHo*!XnJinn?uVNv(@+`yS}yaTdf80B0odEKj2K(CpL<#t`PW*!xYjed??Dj#?6$mAX($byfu-p zCEhKyMsqlk{1v2UI&0X*fk0^1$4(V$0uW2sWwR=~@be_ng`f#Y>rIG)Q}5K6u3Wc3 z@65~>vs36~B4r6oPw9b5QK7A=v@Vix9b7#5B9;@!Qu~zmor>}q>pSyNVAa6&zTwrY zs*_Y@N=kPs489Pf+=K}~O&a6GN$g%`Kg6@c>x922B)9Y`>pC%k)p`8ZK3!d5T`Qft zPZ#4Nn6<~x7S2`lr4>lJH^&uSU>LOLje}-GkN+ff+RP`BPG3^rOdsPJPiL$1AvKW} zqgT_R^v?6TMHM!`Jaw%fo_cWsn*5nqMFhQLn|RfFf4^?lE4;&FQs&^xd?t}yxr^-i zZo=r;RjPP0LWW|kR;l%A<7r34=6JEiZn57zYE}C6Vyjhc9E$ii z#dhg+vAvfEk$3uWut+z2Zy;|aznol*U|cWe=~}aKn8&LP=*ejp-{x^g&mdi@HD9ub zhI)L7Pb~f=D!Tj`awT`4rShv482|WVe^343X?pGE+iE#K!d#$mbo00ZJ{~RbxpG)(%irw^6a`w< zdAb9I)-b6Hr2_MT##I}|_EA58CLYVG_eXcDb+n4=Ev;{Vuh(#p@_$~Oi|k%{TD6Yv zoh*u76OBKee>EH0sPXMi@YDGVxQ{nStywD8Hps^&6LdrKt(ZVknJUv_4qGcmbA+FK zz!h~|&|!AhfUjJDaXAP3Qc4B9`o9MAjEo_e{N2}uSa!5L)mG(jOW7#HxXNSTg=hR@ z>~&kcu9h!QHF&J$RqB|Tf!{DBlXpzyd?mzU0C zrkB2+L8o3{U&Vb)>1rHv-bqI3%?H9l&n9KZ`dozZre#yiD=1%w@lnj9Gqj-O<%x?L za|vV&DzTW&%IQM#x5R#KqXZ8(3@QT}39UUlW&P@9m4P+G1nGX6kLv8oC04PWU;2y4 zUS5K1eO$z?Q;;(|wAUfxuj1_EDK#6fs)yF%M$@6UoVfq<09!Who%-TS>=J|@+ib&Z zv=Q^&(1n79WDFYy)4fyqt%qj?;E{v|{Ffa~&!u0BhxqnQwel7dwX27~cHl{D8^=`b zW~ox{wJ}|tC+r+GnvJ76*H3862X5G&pfYnwL>wQyzmB0PqEujfFYRwiPfKBm?Z&*>@f(Kx6pdDcT{nLvn!*UoFseZ@ zKWE3J6fN*h{YdMzs2IgnFi2%b%cxXx#*g}^7hrL4-ULZAe>j|{>#lAO$` zf5JGK%yicO2n=ns2s1_`L*)KAqbKrY`fvh+3=nS$r3UjDV4pBo%{DH&UDAZv%{iKL7{ zZ?q7hv&CdQ!Rw{8-7aV72f{6Ve9tXyrx0ga=Ck?G*dua##{=z{9svW1314K${_pVp z|2A;s?cV>iOZX4Q`QP00-Dv;+eT3-z@9ob220(`2=$!7FgSoR?9n6JXs&wzUTp4LU zGi%T5JC$pcq8*8y;knP-RA4!i8&t=-Be`qqr7W83LepZ_9rQIC zN^gCK7FhzA()9Lbr*>!SPVJhFv>n>@wrd-Qc4ziOyIx|#&g{;D&sCa}=kBL@@9D^H z$bsGP+c zhPg>cxGPZA6fqi%$emkg-kZgt-ql6z!3}>?ZFhUc8cf|7Ihz;983`|o{jwG02Zpwfu*r-q<2+w;6PT4q;&-am zX-V__Tu2{ '' THEN c.domain_name -- in case of datatype is a domain name expected type is domain_name e.g "lo" is a domain name over oid https://www.postgresql.org/docs/current/lo.html#LO-RATIONALE:~:text=The%20module%20also%20provides%20a%20data%20type%20lo%2C%20which%20is%20really%20just%20a%20domain%20over%20the%20oid%20type ELSE c.data_type -- in native type cases using data_type END AS data_type FROM diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 5c6b62da27..686e367e11 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -50,8 +50,8 @@ const PG_STAT_STATEMENTS = "pg_stat_statements" var pg_catalog_tables_required = []string{"regclass", "pg_class", "pg_inherits", "setval", "pg_index", "pg_relation_size", "pg_namespace", "pg_tables", "pg_sequences", "pg_roles", "pg_database", "pg_extension"} var information_schema_tables_required = []string{"schemata", "tables", "columns", "key_column_usage", "sequences"} -var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID"} -var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML"} +var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID", "LO"} +var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "LO"} func GetPGLiveMigrationUnsupportedDatatypes() []string { liveMigrationUnsupportedDataTypes, _ := lo.Difference(PostgresUnsupportedDataTypesForDbzm, PostgresUnsupportedDataTypes) From 8ecc7aa96e2dfb113fb06ffc295e81386539d37f Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:48:02 +0530 Subject: [PATCH 067/105] Integrated CheckRequiredToolsAreInstalled() function into checkDependenciesForExport() function (#2071) --- yb-voyager/cmd/export.go | 28 ++++++++++++++++++++++++---- yb-voyager/cmd/exportData.go | 5 ++--- yb-voyager/cmd/exportSchema.go | 1 - yb-voyager/src/srcdb/mysql.go | 4 ---- yb-voyager/src/srcdb/oracle.go | 4 ---- yb-voyager/src/srcdb/postgres.go | 4 ---- yb-voyager/src/srcdb/srcdb.go | 1 - yb-voyager/src/srcdb/utils.go | 15 --------------- yb-voyager/src/srcdb/yugabytedb.go | 4 ---- yb-voyager/src/utils/utils.go | 16 ++++++++++++++++ 10 files changed, 42 insertions(+), 40 deletions(-) diff --git a/yb-voyager/cmd/export.go b/yb-voyager/cmd/export.go index 4f03cfe842..516b80d206 100644 --- a/yb-voyager/cmd/export.go +++ b/yb-voyager/cmd/export.go @@ -43,7 +43,7 @@ var useDebezium bool var runId string var excludeTableListFilePath string var tableListFilePath string -var pgExportDependencies = []string{"pg_dump", "pg_restore", "psql"} +var pgExportCommands = []string{"pg_dump", "pg_restore", "psql"} var exportCmd = &cobra.Command{ Use: "export", @@ -392,9 +392,11 @@ func saveExportTypeInMSR() { } func checkDependenciesForExport() (binaryCheckIssues []string, err error) { - if source.DBType == POSTGRESQL { + var missingTools []string + switch source.DBType { + case POSTGRESQL: sourceDBVersion := source.DB().GetVersion() - for _, binary := range pgExportDependencies { + for _, binary := range pgExportCommands { _, binaryCheckIssue, err := srcdb.GetAbsPathOfPGCommandAboveVersion(binary, sourceDBVersion) if err != nil { return nil, err @@ -402,8 +404,26 @@ func checkDependenciesForExport() (binaryCheckIssues []string, err error) { binaryCheckIssues = append(binaryCheckIssues, binaryCheckIssue) } } + + missingTools = utils.CheckTools("strings") + + case MYSQL: + // TODO: For mysql and oracle, we can probably remove the ora2pg check in case it is a live migration + // Issue Link: https://github.com/yugabyte/yb-voyager/issues/2102 + missingTools = utils.CheckTools("ora2pg") + + case ORACLE: + missingTools = utils.CheckTools("ora2pg", "sqlplus") + + case YUGABYTEDB: + missingTools = utils.CheckTools("strings") + + default: + return nil, fmt.Errorf("unknown source database type %q", source.DBType) } + binaryCheckIssues = append(binaryCheckIssues, missingTools...) + if changeStreamingIsEnabled(exportType) || useDebezium { // Check for java javaIssue, err := checkJavaVersion() @@ -489,7 +509,7 @@ func checkJavaVersion() (binaryCheckIssue string, err error) { } if majorVersion < MIN_REQUIRED_JAVA_VERSION { - return fmt.Sprintf("Java version %s is not supported. Please install Java version %d or higher", version, MIN_REQUIRED_JAVA_VERSION), nil + return fmt.Sprintf("java: required version >= %d; current version: %s", MIN_REQUIRED_JAVA_VERSION, version), nil } return "", nil diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index 14cd85ab0f..3016017843 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -222,8 +222,8 @@ func exportData() bool { if err != nil { utils.ErrExit("check dependencies for export: %v", err) } else if len(binaryCheckIssues) > 0 { - color.Red("\nMissing dependencies for export data:") - utils.PrintAndLog("\n%s", strings.Join(binaryCheckIssues, "\n")) + headerStmt := color.RedString("Missing dependencies for export data:") + utils.PrintAndLog("\n%s\n%s", headerStmt, strings.Join(binaryCheckIssues, "\n")) utils.ErrExit("") } } @@ -245,7 +245,6 @@ func exportData() bool { clearMigrationStateIfRequired() checkSourceDBCharset() - source.DB().CheckRequiredToolsAreInstalled() saveSourceDBConfInMSR() saveExportTypeInMSR() err = InitNameRegistry(exportDir, exporterRole, &source, source.DB(), nil, nil, false) diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index 3625129955..4adcdca1ac 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -118,7 +118,6 @@ func exportSchema() error { } checkSourceDBCharset() - source.DB().CheckRequiredToolsAreInstalled() sourceDBVersion := source.DB().GetVersion() source.DBVersion = sourceDBVersion source.DBSize, err = source.DB().GetDatabaseSize() diff --git a/yb-voyager/src/srcdb/mysql.go b/yb-voyager/src/srcdb/mysql.go index 97ab62e47c..a30dce839e 100644 --- a/yb-voyager/src/srcdb/mysql.go +++ b/yb-voyager/src/srcdb/mysql.go @@ -71,10 +71,6 @@ func (ms *MySQL) CheckSchemaExists() bool { return true } -func (ms *MySQL) CheckRequiredToolsAreInstalled() { - checkTools("ora2pg") -} - func (ms *MySQL) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.AsQualifiedCatalogName()) diff --git a/yb-voyager/src/srcdb/oracle.go b/yb-voyager/src/srcdb/oracle.go index a1fbd634fc..5527963ec7 100644 --- a/yb-voyager/src/srcdb/oracle.go +++ b/yb-voyager/src/srcdb/oracle.go @@ -78,10 +78,6 @@ func (ora *Oracle) CheckSchemaExists() bool { return true } -func (ora *Oracle) CheckRequiredToolsAreInstalled() { - checkTools("ora2pg", "sqlplus") -} - func (ora *Oracle) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery()) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 686e367e11..34249fafda 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -140,10 +140,6 @@ func (pg *PostgreSQL) getTrimmedSchemaList() []string { return trimmedList } -func (pg *PostgreSQL) CheckRequiredToolsAreInstalled() { - checkTools("strings") -} - func (pg *PostgreSQL) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery()) diff --git a/yb-voyager/src/srcdb/srcdb.go b/yb-voyager/src/srcdb/srcdb.go index 1c375128b8..df707dcdab 100644 --- a/yb-voyager/src/srcdb/srcdb.go +++ b/yb-voyager/src/srcdb/srcdb.go @@ -33,7 +33,6 @@ type SourceDB interface { GetConnectionUriWithoutPassword() string GetTableRowCount(tableName sqlname.NameTuple) (int64, error) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 - CheckRequiredToolsAreInstalled() GetVersion() string GetAllTableNames() []*sqlname.SourceName GetAllTableNamesRaw(schemaName string) ([]string, error) diff --git a/yb-voyager/src/srcdb/utils.go b/yb-voyager/src/srcdb/utils.go index 30f03df904..12aaf374a3 100644 --- a/yb-voyager/src/srcdb/utils.go +++ b/yb-voyager/src/srcdb/utils.go @@ -17,25 +17,10 @@ limitations under the License. import ( "fmt" "os" - "os/exec" "path" "strings" - - log "github.com/sirupsen/logrus" - - "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) -func checkTools(tools ...string) { - for _, tool := range tools { - execPath, err := exec.LookPath(tool) - if err != nil { - utils.ErrExit("%q not found. Check if it is installed and included in the path.", tool) - } - log.Infof("Found %q", execPath) - } -} - func findAllExecutablesInPath(executableName string) ([]string, error) { pathString := os.Getenv("PATH") if pathString == "" { diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index 8b654a21a0..c2c7ef07e5 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -70,10 +70,6 @@ func (yb *YugabyteDB) Disconnect() { } } -func (yb *YugabyteDB) CheckRequiredToolsAreInstalled() { - checkTools("strings") -} - func (yb *YugabyteDB) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) { var rowCount int64 query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery()) diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index c960814b9b..ecf178a575 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -24,6 +24,7 @@ import ( "net" "net/url" "os" + "os/exec" "path/filepath" "regexp" "sort" @@ -724,3 +725,18 @@ func GetFinalReleaseVersionFromRCVersion(msrVoyagerFinalVersion string) (string, } return msrVoyagerFinalVersion, nil } + +// Return list of missing tools from the provided list of tools +func CheckTools(tools ...string) []string { + var missingTools []string + for _, tool := range tools { + execPath, err := exec.LookPath(tool) + if err != nil { + missingTools = append(missingTools, tool) + } else { + log.Infof("Found %s at %s", tool, execPath) + } + } + + return missingTools +} From ff8bf66dde06c8fa70415942efb8b55612dcc771 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 19 Dec 2024 17:09:39 +0530 Subject: [PATCH 068/105] Fix: Not logging the source-replica-db-password in the import-data-to-source-replica command log file (#2105) --- yb-voyager/cmd/logging.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/cmd/logging.go b/yb-voyager/cmd/logging.go index c28b47c28d..21b74ecee2 100644 --- a/yb-voyager/cmd/logging.go +++ b/yb-voyager/cmd/logging.go @@ -81,7 +81,7 @@ func InitLogging(logDir string, logLevel string, disableLogging bool, cmdName st func redactPasswordFromArgs() { for i := 0; i < len(os.Args); i++ { opt := os.Args[i] - if opt == "--source-db-password" || opt == "--target-db-password" || opt == "--ff-db-password" { + if opt == "--source-db-password" || opt == "--target-db-password" || opt == "--source-replica-db-password" { os.Args[i+1] = "XXX" } } From b9489425685cf16b1729c4b305ea51b44b80e691 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Mon, 23 Dec 2024 11:12:44 +0530 Subject: [PATCH 069/105] Updated code to use pg_query_go v6 instead of v5 (#2103) --- .../expected_files/expectedAssessmentReport.json | 7 ++++++- .../expected_schema_analysis_report.json | 14 +++++++++++++- yb-voyager/cmd/analyzeSchema.go | 2 +- yb-voyager/cmd/exportSchema.go | 2 +- yb-voyager/go.mod | 4 ++-- yb-voyager/go.sum | 4 ++-- yb-voyager/src/query/queryparser/ddl_processor.go | 2 +- .../src/query/queryparser/helpers_protomsg.go | 2 +- yb-voyager/src/query/queryparser/helpers_struct.go | 2 +- yb-voyager/src/query/queryparser/query_parser.go | 2 +- 10 files changed, 29 insertions(+), 12 deletions(-) diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 8aa82381e4..082f0d5018 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -36,7 +36,7 @@ { "ObjectType": "FUNCTION", "TotalCount": 139, - "InvalidCount": 16, + "InvalidCount": 17, "ObjectNames": "mgd.acc_accession_delete, mgd.acc_assignj, mgd.acc_assignmgi, mgd.acc_delete_byacckey, mgd.acc_insert, mgd.acc_setmax, mgd.acc_split, mgd.acc_update, mgd.accref_insert, mgd.accref_process, mgd.all_allele_delete, mgd.all_allele_insert, mgd.all_allele_update, mgd.all_cellline_delete, mgd.all_cellline_update1, mgd.all_cellline_update2, mgd.all_convertallele, mgd.all_createwildtype, mgd.all_insertallele, mgd.all_mergeallele, mgd.all_mergewildtypes, mgd.all_reloadlabel, mgd.all_variant_delete, mgd.bib_keepwfrelevance, mgd.bib_refs_delete, mgd.bib_refs_insert, mgd.bib_reloadcache, mgd.bib_updatewfstatusap, mgd.bib_updatewfstatusgo, mgd.bib_updatewfstatusgxd, mgd.bib_updatewfstatusqtl, mgd.gxd_addcelltypeset, mgd.gxd_addemapaset, mgd.gxd_addgenotypeset, mgd.gxd_allelepair_insert, mgd.gxd_antibody_delete, mgd.gxd_antibody_insert, mgd.gxd_antigen_delete, mgd.gxd_antigen_insert, mgd.gxd_assay_delete, mgd.gxd_assay_insert, mgd.gxd_checkduplicategenotype, mgd.gxd_gelrow_insert, mgd.gxd_genotype_delete, mgd.gxd_genotype_insert, mgd.gxd_getgenotypesdatasets, mgd.gxd_getgenotypesdatasetscount, mgd.gxd_htexperiment_delete, mgd.gxd_htrawsample_delete, mgd.gxd_htsample_ageminmax, mgd.gxd_index_insert, mgd.gxd_index_insert_before, mgd.gxd_index_update, mgd.gxd_orderallelepairs, mgd.gxd_ordergenotypes, mgd.gxd_ordergenotypesall, mgd.gxd_ordergenotypesmissing, mgd.gxd_removebadgelband, mgd.gxd_replacegenotype, mgd.img_image_delete, mgd.img_image_insert, mgd.img_setpdo, mgd.mgi_addsetmember, mgd.mgi_checkemapaclipboard, mgd.mgi_cleannote, mgd.mgi_deleteemapaclipboarddups, mgd.mgi_insertreferenceassoc, mgd.mgi_insertsynonym, mgd.mgi_mergerelationship, mgd.mgi_organism_delete, mgd.mgi_organism_insert, mgd.mgi_processnote, mgd.mgi_reference_assoc_delete, mgd.mgi_reference_assoc_insert, mgd.mgi_relationship_delete, mgd.mgi_resetageminmax, mgd.mgi_resetsequencenum, mgd.mgi_setmember_emapa_insert, mgd.mgi_setmember_emapa_update, mgd.mgi_statistic_delete, mgd.mgi_updatereferenceassoc, mgd.mgi_updatesetmember, mgd.mld_expt_marker_update, mgd.mld_expts_delete, mgd.mld_expts_insert, mgd.mrk_allelewithdrawal, mgd.mrk_cluster_delete, mgd.mrk_copyhistory, mgd.mrk_deletewithdrawal, mgd.mrk_inserthistory, mgd.mrk_marker_delete, mgd.mrk_marker_insert, mgd.mrk_marker_update, mgd.mrk_mergewithdrawal, mgd.mrk_reloadlocation, mgd.mrk_reloadreference, mgd.mrk_simplewithdrawal, mgd.mrk_strainmarker_delete, mgd.mrk_updatekeys, mgd.prb_ageminmax, mgd.prb_getstrainbyreference, mgd.prb_getstraindatasets, mgd.prb_getstrainreferences, mgd.prb_insertreference, mgd.prb_marker_insert, mgd.prb_marker_update, mgd.prb_mergestrain, mgd.prb_probe_delete, mgd.prb_probe_insert, mgd.prb_processanonymoussource, mgd.prb_processprobesource, mgd.prb_processseqloadersource, mgd.prb_processsequencesource, mgd.prb_reference_delete, mgd.prb_reference_update, mgd.prb_setstrainreview, mgd.prb_source_delete, mgd.prb_strain_delete, mgd.prb_strain_insert, mgd.seq_deletebycreatedby, mgd.seq_deletedummy, mgd.seq_deleteobsoletedummy, mgd.seq_merge, mgd.seq_sequence_delete, mgd.seq_split, mgd.voc_annot_insert, mgd.voc_copyannotevidencenotes, mgd.voc_evidence_delete, mgd.voc_evidence_insert, mgd.voc_evidence_property_delete, mgd.voc_evidence_update, mgd.voc_mergeannotations, mgd.voc_mergedupannotations, mgd.voc_mergeterms, mgd.voc_processannotheader, mgd.voc_processannotheaderall, mgd.voc_processannotheadermissing, mgd.voc_resetterms, mgd.voc_term_delete" }, { @@ -13780,6 +13780,11 @@ "ObjectType": "FUNCTION", "ObjectName": "mgd.seq_split", "SqlStatement": "acc_accession.accID%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mgi_resetageminmax", + "SqlStatement": "prb_source.age%TYPE" } ], "MinimumVersionsFixedIn": null, diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index cfe0307c16..bc5c2eee99 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -36,7 +36,7 @@ { "ObjectType": "FUNCTION", "TotalCount": 139, - "InvalidCount": 16, + "InvalidCount": 17, "ObjectNames": "mgd.acc_accession_delete, mgd.acc_assignj, mgd.acc_assignmgi, mgd.acc_delete_byacckey, mgd.acc_insert, mgd.acc_setmax, mgd.acc_split, mgd.acc_update, mgd.accref_insert, mgd.accref_process, mgd.all_allele_delete, mgd.all_allele_insert, mgd.all_allele_update, mgd.all_cellline_delete, mgd.all_cellline_update1, mgd.all_cellline_update2, mgd.all_convertallele, mgd.all_createwildtype, mgd.all_insertallele, mgd.all_mergeallele, mgd.all_mergewildtypes, mgd.all_reloadlabel, mgd.all_variant_delete, mgd.bib_keepwfrelevance, mgd.bib_refs_delete, mgd.bib_refs_insert, mgd.bib_reloadcache, mgd.bib_updatewfstatusap, mgd.bib_updatewfstatusgo, mgd.bib_updatewfstatusgxd, mgd.bib_updatewfstatusqtl, mgd.gxd_addcelltypeset, mgd.gxd_addemapaset, mgd.gxd_addgenotypeset, mgd.gxd_allelepair_insert, mgd.gxd_antibody_delete, mgd.gxd_antibody_insert, mgd.gxd_antigen_delete, mgd.gxd_antigen_insert, mgd.gxd_assay_delete, mgd.gxd_assay_insert, mgd.gxd_checkduplicategenotype, mgd.gxd_gelrow_insert, mgd.gxd_genotype_delete, mgd.gxd_genotype_insert, mgd.gxd_getgenotypesdatasets, mgd.gxd_getgenotypesdatasetscount, mgd.gxd_htexperiment_delete, mgd.gxd_htrawsample_delete, mgd.gxd_htsample_ageminmax, mgd.gxd_index_insert, mgd.gxd_index_insert_before, mgd.gxd_index_update, mgd.gxd_orderallelepairs, mgd.gxd_ordergenotypes, mgd.gxd_ordergenotypesall, mgd.gxd_ordergenotypesmissing, mgd.gxd_removebadgelband, mgd.gxd_replacegenotype, mgd.img_image_delete, mgd.img_image_insert, mgd.img_setpdo, mgd.mgi_addsetmember, mgd.mgi_checkemapaclipboard, mgd.mgi_cleannote, mgd.mgi_deleteemapaclipboarddups, mgd.mgi_insertreferenceassoc, mgd.mgi_insertsynonym, mgd.mgi_mergerelationship, mgd.mgi_organism_delete, mgd.mgi_organism_insert, mgd.mgi_processnote, mgd.mgi_reference_assoc_delete, mgd.mgi_reference_assoc_insert, mgd.mgi_relationship_delete, mgd.mgi_resetageminmax, mgd.mgi_resetsequencenum, mgd.mgi_setmember_emapa_insert, mgd.mgi_setmember_emapa_update, mgd.mgi_statistic_delete, mgd.mgi_updatereferenceassoc, mgd.mgi_updatesetmember, mgd.mld_expt_marker_update, mgd.mld_expts_delete, mgd.mld_expts_insert, mgd.mrk_allelewithdrawal, mgd.mrk_cluster_delete, mgd.mrk_copyhistory, mgd.mrk_deletewithdrawal, mgd.mrk_inserthistory, mgd.mrk_marker_delete, mgd.mrk_marker_insert, mgd.mrk_marker_update, mgd.mrk_mergewithdrawal, mgd.mrk_reloadlocation, mgd.mrk_reloadreference, mgd.mrk_simplewithdrawal, mgd.mrk_strainmarker_delete, mgd.mrk_updatekeys, mgd.prb_ageminmax, mgd.prb_getstrainbyreference, mgd.prb_getstraindatasets, mgd.prb_getstrainreferences, mgd.prb_insertreference, mgd.prb_marker_insert, mgd.prb_marker_update, mgd.prb_mergestrain, mgd.prb_probe_delete, mgd.prb_probe_insert, mgd.prb_processanonymoussource, mgd.prb_processprobesource, mgd.prb_processseqloadersource, mgd.prb_processsequencesource, mgd.prb_reference_delete, mgd.prb_reference_update, mgd.prb_setstrainreview, mgd.prb_source_delete, mgd.prb_strain_delete, mgd.prb_strain_insert, mgd.seq_deletebycreatedby, mgd.seq_deletedummy, mgd.seq_deleteobsoletedummy, mgd.seq_merge, mgd.seq_sequence_delete, mgd.seq_split, mgd.voc_annot_insert, mgd.voc_copyannotevidencenotes, mgd.voc_evidence_delete, mgd.voc_evidence_insert, mgd.voc_evidence_property_delete, mgd.voc_evidence_update, mgd.voc_mergeannotations, mgd.voc_mergedupannotations, mgd.voc_mergeterms, mgd.voc_processannotheader, mgd.voc_processannotheaderall, mgd.voc_processannotheadermissing, mgd.voc_resetterms, mgd.voc_term_delete" }, { @@ -1325,6 +1325,18 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mgi_resetageminmax", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "prb_source.age%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 0c8d21cc64..80a4d14627 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -27,7 +27,7 @@ import ( "strings" "text/template" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index 4adcdca1ac..092a87c39e 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -22,7 +22,7 @@ import ( "strings" "github.com/fatih/color" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index c833375e2f..a8a306dbcb 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -11,6 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 github.com/davecgh/go-spew v1.1.1 + github.com/deckarep/golang-set/v2 v2.7.0 github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 @@ -29,7 +30,7 @@ require ( github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 github.com/mitchellh/go-ps v1.0.0 github.com/nightlyone/lockfile v1.0.0 - github.com/pganalyze/pg_query_go/v5 v5.1.0 + github.com/pganalyze/pg_query_go/v6 v6.0.0 github.com/samber/lo v1.38.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 @@ -55,7 +56,6 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect - github.com/deckarep/golang-set/v2 v2.7.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 44ee2af8de..3b156d1c04 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1711,8 +1711,8 @@ github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwb github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg= -github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug= +github.com/pganalyze/pg_query_go/v6 v6.0.0 h1:in6RkR/apfqlAtvqgDxd4Y4o87a5Pr8fkKDB4DrDo2c= +github.com/pganalyze/pg_query_go/v6 v6.0.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 22f60fd48d..2d2a6e7665 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -20,7 +20,7 @@ import ( "slices" "strings" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" ) diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 88b3a85d64..0eb609d15f 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -18,7 +18,7 @@ package queryparser import ( "strings" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" ) diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 67b4294af0..eafc907910 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" ) diff --git a/yb-voyager/src/query/queryparser/query_parser.go b/yb-voyager/src/query/queryparser/query_parser.go index 7246436830..864cb30d07 100644 --- a/yb-voyager/src/query/queryparser/query_parser.go +++ b/yb-voyager/src/query/queryparser/query_parser.go @@ -29,7 +29,7 @@ package queryparser import ( "fmt" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" ) From 47fe19820f384a5dad17a99318a676c4ec55e399 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 23 Dec 2024 15:59:24 +0530 Subject: [PATCH 070/105] Add regex functions issue (#2100) - Detect and report unsupported regex functions --- .github/workflows/misc-migtests.yml | 3 +- .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 14 ++++- .../pg_assessment_report.sql | 5 +- .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 6 +- .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/cmd/assessMigrationCommand.go | 63 ++++++++++--------- yb-voyager/cmd/constants.go | 1 + yb-voyager/src/query/queryissue/constants.go | 2 + yb-voyager/src/query/queryissue/detectors.go | 10 ++- yb-voyager/src/query/queryissue/helpers.go | 4 ++ yb-voyager/src/query/queryissue/issues_dml.go | 13 ++++ .../src/query/queryissue/issues_dml_test.go | 22 +++++++ .../queryissue/parser_issue_detector_test.go | 30 +++++++++ yb-voyager/src/utils/commonVariables.go | 3 +- 23 files changed, 184 insertions(+), 38 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index add010cdd9..cd9fe7ad35 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -12,7 +12,7 @@ jobs: services: postgres: - image: postgres:14 + image: postgres:15 env: POSTGRES_PASSWORD: secret # Set health checks to wait until postgres has started @@ -59,6 +59,7 @@ jobs: run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support + echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH" env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 85faaee1cc..509e19e385 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -741,6 +741,11 @@ "FeatureName": "Large Object Functions", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index a767030089..0c9360ba37 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -176,6 +176,11 @@ "FeatureName": "System Columns", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 931e2611e7..c7e3cb30ce 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -45,7 +45,7 @@ { "ObjectType": "TABLE", "TotalCount": 66, - "InvalidCount": 22, + "InvalidCount": 23, "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { @@ -590,6 +590,16 @@ } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [ + { + "ObjectName": "public.ordersentry", + "SqlStatement": "CREATE TABLE public.ordersentry (\n order_id integer NOT NULL,\n customer_name text NOT NULL,\n product_name text NOT NULL,\n quantity integer NOT NULL,\n price numeric(10,2) NOT NULL,\n processed_at timestamp without time zone,\n r integer DEFAULT regexp_count('This is an example. Another example. Example is a common word.'::text, 'example'::text)\n);" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1158,7 +1168,7 @@ "SchemaName": "public", "ObjectName": "ordersentry", "RowCount": 6, - "ColumnCount": 6, + "ColumnCount": 7, "Reads": 0, "Writes": 6, "ReadsPerSecond": 0, diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index b08d4f1acc..0c089922b5 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -242,7 +242,8 @@ EXECUTE FUNCTION public.check_sales_region(); product_name TEXT NOT NULL, quantity INT NOT NULL, price NUMERIC(10, 2) NOT NULL, - processed_at timestamp + processed_at timestamp, + r INT DEFAULT regexp_count('This is an example. Another example. Example is a common word.', 'example') -- regex functions in default ); INSERT INTO public.ordersentry (customer_name, product_name, quantity, price) @@ -360,4 +361,4 @@ BEGIN PERFORM lo_unlink(loid); END IF; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 082f0d5018..329c9404b5 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -622,6 +622,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 5b804b142c..a553ba272f 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -683,6 +683,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index ebc2540296..b03097dacc 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -209,6 +209,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index 65c1989f28..dcea98ff79 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -204,6 +204,11 @@ "FeatureName": "View with check option", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 945e5e2039..3178c50abd 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -943,8 +943,12 @@ "FeatureName": "Large Object Functions", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } - ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index d2a1b9165f..ed5bc11f99 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -274,6 +274,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 141da784d1..0c32b640ed 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -206,6 +206,11 @@ "FeatureName": "View with check option", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 412bcdf959..4bef6f17a3 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -384,6 +384,11 @@ "FeatureName": "Large Object Functions", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 80a4d14627..85d1ce25c7 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -688,6 +688,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil ObjectType: issueInstance.ObjectType, ObjectName: displayObjectName, Reason: issueInstance.TypeName, + Type: issueInstance.Type, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 305ec56535..21e006c081 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -947,7 +947,7 @@ func areMinVersionsFixedInEqual(m1 map[string]*ybversion.YBVersion, m2 map[strin return true } -func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { +func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, issueType string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { log.Info("filtering issues for feature: ", featureName) objects := make([]ObjectInfo, 0) link := "" // for oracle we shouldn't display any line for links @@ -958,7 +958,9 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, issue.IssueType) { continue } - if strings.Contains(issue.Reason, issueReason) { + issueMatched := lo.Ternary[bool](issueType != "", issueType == issue.Type, strings.Contains(issue.Reason, issueReason)) + + if issueMatched { objectInfo := ObjectInfo{ ObjectName: issue.ObjectName, SqlStatement: issue.SqlStatement, @@ -984,33 +986,34 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem displayIndexMethod := strings.ToUpper(indexMethod) feature := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(feature, reason, schemaAnalysisReport, false, "")) - } - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, STORED_GENERATED_COLUMN_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, GIN_INDEX_MULTI_COLUMN_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, ALTER_TABLE_SET_ATTRIBUTE_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, ALTER_TABLE_DISABLE_RULE_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, ALTER_TABLE_CLUSTER_ON_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, STORAGE_PARAMETERS_DDL_STMT_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, EXCLUSION_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(feature, reason, "", schemaAnalysisReport, false, "")) + } + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, STORED_GENERATED_COLUMN_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, GIN_INDEX_MULTI_COLUMN_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, ALTER_TABLE_SET_ATTRIBUTE_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, ALTER_TABLE_DISABLE_RULE_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, ALTER_TABLE_CLUSTER_ON_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, STORAGE_PARAMETERS_DDL_STMT_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, EXCLUSION_CONSTRAINT_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)) pkOrUkConstraintIssuePrefix := strings.Split(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, "%s")[0] - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, queryissue.ADVISORY_LOCKS_NAME, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, queryissue.XML_FUNCTIONS_NAME, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, queryissue.ADVISORY_LOCKS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, queryissue.XML_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) return unsupportedFeatures, nil } @@ -1025,7 +1028,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. for _, unsupportedType := range unsupportedIndexDatatypes { - indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), schemaAnalysisReport, false, "") + indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") for _, object := range indexes.Objects { formattedObject := object formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) @@ -1042,7 +1045,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem func fetchUnsupportedOracleFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for Oracle...") unsupportedFeatures := make([]UnsupportedFeature, 0) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) return unsupportedFeatures, nil } @@ -1382,11 +1385,11 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration case POSTGRESQL: log.Infof("add migration caveats to assessment report") migrationCaveats := make([]UnsupportedFeature, 0) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, "", schemaAnalysisReport, true, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, FOREIGN_TABLE_ISSUE_REASON, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, FOREIGN_TABLE_ISSUE_REASON, "", schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, POLICY_ROLE_ISSUE, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, POLICY_ROLE_ISSUE, "", schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_ISSUE)) if len(unsupportedDataTypesForLiveMigration) > 0 { diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 85c9f225e4..b22891ae06 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -213,6 +213,7 @@ const ( REFERENCING_TRIGGER_FEATURE = "REFERENCING clause for triggers" BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables" PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes" + REGEX_FUNCTIONS_FEATURE = "Regex Functions" // Migration caveats diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index e07367a38b..3a6d424538 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -59,6 +59,8 @@ const ( ADVISORY_LOCKS_NAME = "Advisory Locks" SYSTEM_COLUMNS_NAME = "System Columns" XML_FUNCTIONS_NAME = "XML Functions" + + REGEX_FUNCTIONS = "REGEX_FUNCTIONS" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 228496a69e..bb9490786f 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -32,9 +32,10 @@ type UnsupportedConstructDetector interface { type FuncCallDetector struct { query string - // right now it covers Advisory Locks and XML functions + advisoryLocksFuncsDetected mapset.Set[string] xmlFuncsDetected mapset.Set[string] + regexFuncsDetected mapset.Set[string] loFuncsDetected mapset.Set[string] } @@ -43,6 +44,7 @@ func NewFuncCallDetector(query string) *FuncCallDetector { query: query, advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](), xmlFuncsDetected: mapset.NewThreadUnsafeSet[string](), + regexFuncsDetected: mapset.NewThreadUnsafeSet[string](), loFuncsDetected: mapset.NewThreadUnsafeSet[string](), } } @@ -62,6 +64,9 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) error { if unsupportedXmlFunctions.ContainsOne(funcName) { d.xmlFuncsDetected.Add(funcName) } + if unsupportedRegexFunctions.ContainsOne(funcName) { + d.regexFuncsDetected.Add(funcName) + } if unsupportedLargeObjectFunctions.ContainsOne(funcName) { d.loFuncsDetected.Add(funcName) @@ -78,6 +83,9 @@ func (d *FuncCallDetector) GetIssues() []QueryIssue { if d.xmlFuncsDetected.Cardinality() > 0 { issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } + if d.regexFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } if d.loFuncsDetected.Cardinality() > 0 { issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index 01bacd4b7c..617b2fb701 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -58,6 +58,10 @@ var unsupportedXmlFunctions = mapset.NewThreadUnsafeSet([]string{ "xmlconcat2", "xmlvalidate", "xml_in", "xml_out", "xml_recv", "xml_send", // System XML I/O }...) +var unsupportedRegexFunctions = mapset.NewThreadUnsafeSet([]string{ + "regexp_count", "regexp_instr", "regexp_like", +}...) + var UnsupportedIndexMethods = []string{ "gist", "brin", diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 3528ead45d..33db98be0f 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -57,6 +57,19 @@ func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement str return newQueryIssue(xmlFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } +var regexFunctionsIssue = issue.Issue{ + Type: REGEX_FUNCTIONS, + TypeName: "Regex Functions", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(regexFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var loFunctionsIssue = issue.Issue{ Type: LARGE_OBJECT_FUNCTIONS, TypeName: LARGE_OBJECT_FUNCTIONS_NAME, diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 8f7c6610da..af3d8afcc1 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -40,6 +40,25 @@ func testLOFunctionsIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "Transaction for catalog table write operation 'pg_largeobject_metadata' not found", loDatatypeIssue) } +func testRegexFunctionsIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + + stmts := []string{ + `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_instr('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_like('This is an example. Another example. Example is a common word.', 'example')`, + } + + for _, stmt := range stmts { + _, err = conn.Exec(ctx, stmt) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", regexFunctionsIssue) + } +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -68,4 +87,7 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success := t.Run(fmt.Sprintf("%s-%s", "lo functions", ybVersion), testLOFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 9ba4dac4d1..e7c979118f 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -485,3 +485,33 @@ func TestSingleXMLIssueIsDetected(t *testing.T) { fatalIfError(t, err) assert.Equal(t, 1, len(issues)) } + +func TestRegexFunctionsIssue(t *testing.T) { + dmlStmts := []string{ + `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_instr('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_like('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_count('abc','abc'), regexp_instr('abc','abc'), regexp_like('abc','abc')`, + } + + ddlStmts := []string{ + `CREATE TABLE x (id INT PRIMARY KEY, id2 INT DEFAULT regexp_count('This is an example. Another example. Example is a common word.', 'example'))`, + } + + parserIssueDetector := NewParserIssueDetector() + + for _, stmt := range dmlStmts { + issues, err := parserIssueDetector.getDMLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, 1, len(issues)) + assert.Equal(t, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", stmt), issues[0]) + } + + for _, stmt := range ddlStmts { + issues, err := parserIssueDetector.getDDLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, 1, len(issues)) + assert.Equal(t, NewRegexFunctionsIssue(TABLE_OBJECT_TYPE, "x", stmt), issues[0]) + } + +} diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index 4c18b78fad..c1d51326bc 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -101,10 +101,11 @@ type DBObject struct { // TODO: support MinimumVersionsFixedIn in xml type Issue struct { - IssueType string `json:"IssueType"` + IssueType string `json:"IssueType"` //category: unsupported_features, unsupported_plpgsql_objects, etc ObjectType string `json:"ObjectType"` ObjectName string `json:"ObjectName"` Reason string `json:"Reason"` + Type string `json:"-" xml:"-"` // identifier for issue type ADVISORY_LOCKS, SYSTEM_COLUMNS, etc SqlStatement string `json:"SqlStatement,omitempty"` FilePath string `json:"FilePath"` Suggestion string `json:"Suggestion"` From ecb4c05a15e7d68a6b9e6e64f94d10390b590436 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Mon, 23 Dec 2024 17:39:45 +0530 Subject: [PATCH 071/105] Do not add the feature in UnsupportedFeatures field if there are no objects for it in the JSON report (#2110) --- .../expectedChild2AssessmentReport.json | 28 +--- .../expectedAssessmentReport.json | 141 +--------------- .../expectedAssessmentReport.json | 154 +----------------- .../expectedAssessmentReport.json | 57 ------- .../expectedAssessmentReport.json | 148 +---------------- .../expectedAssessmentReport.json | 103 +----------- .../expectedAssessmentReport.json | 143 +--------------- .../expectedAssessmentReport.json | 145 +---------------- .../expectedAssessmentReport.json | 148 +---------------- .../expectedAssessmentReport.json | 140 +--------------- .../expectedAssessmentReport.json | 145 +---------------- .../expectedAssessmentReport.json | 148 +---------------- yb-voyager/cmd/assessMigrationCommand.go | 33 +++- 13 files changed, 40 insertions(+), 1493 deletions(-) diff --git a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json index 297ad902e4..8e99cbde6b 100644 --- a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json +++ b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json @@ -106,33 +106,7 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": [ - { - "FeatureName": "Compound Triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unsupported Indexes", - "Objects": null, - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Virtual Columns", - "Objects": null, - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited Types", - "Objects": null, - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unsupported Partitioning Methods", - "Objects": null, - "MinimumVersionsFixedIn": null - } - ], + "UnsupportedFeatures": null, "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 509e19e385..90979b4b3f 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -309,66 +309,6 @@ ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Clustering table on index", "Objects": [ @@ -648,16 +588,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, { "FeatureName": "XML Functions", "Objects": [ @@ -696,56 +626,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1774,26 +1654,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index 0c9360ba37..bbcf4613db 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -51,138 +51,7 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": [ - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Advisory Locks", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "XML Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "System Columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } - ], + "UnsupportedFeatures": null, "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { @@ -215,26 +84,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": [ { "ConstructTypeName": "Advisory Locks", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index c7e3cb30ce..edfe82ecd2 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -360,31 +360,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", @@ -397,21 +372,6 @@ "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Exclusion constraints", "Objects": [ @@ -543,11 +503,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "System Columns", "Objects": [ @@ -2009,18 +1964,6 @@ "There are some Unlogged tables in the schema. They will be created as regular LOGGED tables in YugabyteDB as unlogged tables are not supported." ], "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Policies", "Objects": [ diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 329c9404b5..c5d63e0270 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -248,76 +248,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Clustering table on index", "Objects": [ @@ -572,62 +502,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -13541,26 +13416,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": [ { diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index a553ba272f..dd49b1dc67 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -493,21 +493,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, { "FeatureName": "GIST indexes", "Objects": [ @@ -518,26 +503,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited tables", @@ -561,16 +526,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Tables with stored generated columns", @@ -597,26 +552,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Storage parameters in DDLs", @@ -636,21 +571,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "View with check option", @@ -662,11 +582,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Index on complex datatypes", @@ -678,17 +593,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -5553,12 +5458,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Foreign tables", "Objects": [ diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index b03097dacc..3aaf994a31 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -73,21 +73,6 @@ ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, { "FeatureName": "GIST indexes", "Objects": [ @@ -99,76 +84,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Extensions", "Objects": [ @@ -179,42 +94,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -346,26 +226,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index dcea98ff79..3ebfc9a508 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -79,138 +79,7 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } - ], + "UnsupportedFeatures": null, "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { @@ -244,12 +113,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Foreign tables", "Objects": [ @@ -265,12 +128,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 3178c50abd..4294cd126b 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -377,41 +377,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Inherited tables", "Objects": [ @@ -858,97 +823,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -26630,26 +26505,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index ed5bc11f99..8ff126cd0f 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -118,21 +118,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, { "FeatureName": "GIST indexes", "Objects": [ @@ -143,31 +128,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited tables", @@ -199,87 +159,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -986,24 +866,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 0c32b640ed..47797ad8d4 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -76,116 +76,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Exclusion constraints", "Objects": [ @@ -196,22 +86,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -386,24 +261,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 4bef6f17a3..34c9ea429c 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -82,86 +82,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Storage parameters in DDLs", "Objects": [ @@ -344,52 +264,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -1305,26 +1180,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 21e006c081..f67697ae16 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -984,9 +984,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures := make([]UnsupportedFeature, 0) for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) - feature := fmt.Sprintf("%s indexes", displayIndexMethod) + featureName := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(feature, reason, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) } unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, "", schemaAnalysisReport, false, "")) @@ -1015,7 +1015,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) - return unsupportedFeatures, nil + return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { + return len(f.Objects) > 0 + }), nil } func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.SchemaReport, unsupportedIndexDatatypes []string) UnsupportedFeature { @@ -1038,7 +1040,6 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem } } } - return indexesOnComplexTypesFeature } @@ -1046,7 +1047,9 @@ func fetchUnsupportedOracleFeaturesFromSchemaReport(schemaAnalysisReport utils.S log.Infof("fetching unsupported features for Oracle...") unsupportedFeatures := make([]UnsupportedFeature, 0) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - return unsupportedFeatures, nil + return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { + return len(f.Objects) > 0 + }), nil } var OracleUnsupportedIndexTypes = []string{"CLUSTER INDEX", "DOMAIN INDEX", "FUNCTION-BASED DOMAIN INDEX", "IOT - TOP INDEX", "NORMAL/REV INDEX", "FUNCTION-BASED NORMAL/REV INDEX"} @@ -1095,7 +1098,9 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{VIRTUAL_COLUMNS_FEATURE, virtualColumns, false, "", "", nil}) unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{INHERITED_TYPES_FEATURE, inheritedTypes, false, "", "", nil}) unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{UNSUPPORTED_PARTITIONING_METHODS_FEATURE, unsupportedPartitionTypes, false, "", "", nil}) - return unsupportedFeatures, nil + return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { + return len(f.Objects) > 0 + }), nil } func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []UnsupportedFeature { @@ -1397,16 +1402,26 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration for _, col := range unsupportedDataTypesForLiveMigration { columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) + if len(columns) > 0 { + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) + } } if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { columns := make([]ObjectInfo, 0) for _, col := range unsupportedDataTypesForLiveMigrationWithFForFB { columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) + if len(columns) > 0 { + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) + } } - assessmentReport.MigrationCaveats = migrationCaveats + migrationCaveats = lo.Filter(migrationCaveats, func(m UnsupportedFeature, _ int) bool { + return len(m.Objects) > 0 + }) + if len(migrationCaveats) > 0 { + assessmentReport.MigrationCaveats = migrationCaveats + } + } } From 81c30b04af198fedf5585d105c44a4f6ba82bdcf Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 23 Dec 2024 19:10:37 +0530 Subject: [PATCH 072/105] Add --skip-debezium flag in installer script to avoid installing debezium when only offline setup is required (#2109) --- installer_scripts/install-yb-voyager | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/installer_scripts/install-yb-voyager b/installer_scripts/install-yb-voyager index f026999f59..912f3e0697 100755 --- a/installer_scripts/install-yb-voyager +++ b/installer_scripts/install-yb-voyager @@ -12,6 +12,7 @@ LOG_FILE=/tmp/install-yb-voyager.log VERSION="latest" ONLY_PG="false" +SKIP_DEBEZIUM="false" trap on_exit EXIT @@ -323,14 +324,18 @@ check_java() { } install_debezium_server(){ + if [ $SKIP_DEBEZIUM == "true" ] ; then + return + fi + if [ "${VERSION}" != "local" ] then output "Installing debezium:${DEBEZIUM_VERSION}" install_debezium_server_from_release return fi - output "Installing debezium:${VERSION}." + output "Installing debezium:${VERSION}." check_install_maven clean_debezium install_debezium_local ${DEBEZIUM_LOCAL_REF} @@ -538,9 +543,9 @@ rebuild_voyager_local() { get_passed_options() { if [ "$1" == "linux" ] then - OPTS=$(getopt -o "lpvV", --long install-from-local-source,only-pg-support,rebuild-voyager-local,version: --name 'install-yb-voyager' -- $ARGS_LINUX) + OPTS=$(getopt -o "lpdvV", --long install-from-local-source,only-pg-support,skip-debezium,rebuild-voyager-local,version: --name 'install-yb-voyager' -- $ARGS_LINUX) else - OPTS=$(getopt lpvV $ARGS_MACOS) + OPTS=$(getopt lpdvV $ARGS_MACOS) fi eval set -- "$OPTS" @@ -555,6 +560,10 @@ get_passed_options() { ONLY_PG="true"; shift ;; + -d | --skip-debezium ) + SKIP_DEBEZIUM="true" + shift + ;; -v | --rebuild-voyager-local ) REBUILD_VOYAGER_LOCAL="true"; shift From 23033f8a1ed5bf6c2c5f1b85c9f663e0a6561425 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Tue, 24 Dec 2024 16:22:26 +0530 Subject: [PATCH 073/105] Fixed the logic for checking source db version compatibility and added it to assess migration too for PG (#2115) --- yb-voyager/cmd/assessMigrationCommand.go | 8 ++++++++ yb-voyager/src/srcdb/postgres.go | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f67697ae16..0adcff8fe2 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -335,6 +335,14 @@ func assessMigration() (err error) { // We will require source db connection for the below checks // Check if required binaries are installed. if source.RunGuardrailsChecks { + // Check source database version. + log.Info("checking source DB version") + err = source.DB().CheckSourceDBVersion(exportType) + if err != nil { + return fmt.Errorf("source DB version check failed: %w", err) + } + + // Check if required binaries are installed. binaryCheckIssues, err := checkDependenciesForExport() if err != nil { return fmt.Errorf("failed to check dependencies for assess migration: %w", err) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 34249fafda..4c90ffb5e6 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -42,7 +42,7 @@ import ( const MIN_SUPPORTED_PG_VERSION_OFFLINE = "9" const MIN_SUPPORTED_PG_VERSION_LIVE = "10" -const MAX_SUPPORTED_PG_VERSION = "16" +const MAX_SUPPORTED_PG_VERSION = "17" const MISSING = "MISSING" const GRANTED = "GRANTED" const NO_USAGE_PERMISSION = "NO USAGE PERMISSION" @@ -979,14 +979,24 @@ func (pg *PostgreSQL) CheckSourceDBVersion(exportType string) error { if pgVersion == "" { return fmt.Errorf("failed to get source database version") } + + // Extract the major version from the full version string + // Version can be like: 17.2 (Ubuntu 17.2-1.pgdg20.04+1), 17.2, 17alpha1 etc. + re := regexp.MustCompile(`^(\d+)`) + match := re.FindStringSubmatch(pgVersion) + if len(match) < 2 { + return fmt.Errorf("failed to extract major version from source database version: %s", pgVersion) + } + majorVersion := match[1] + supportedVersionRange := fmt.Sprintf("%s to %s", MIN_SUPPORTED_PG_VERSION_OFFLINE, MAX_SUPPORTED_PG_VERSION) - if version.CompareSimple(pgVersion, MAX_SUPPORTED_PG_VERSION) > 0 || version.CompareSimple(pgVersion, MIN_SUPPORTED_PG_VERSION_OFFLINE) < 0 { + if version.CompareSimple(majorVersion, MAX_SUPPORTED_PG_VERSION) > 0 || version.CompareSimple(majorVersion, MIN_SUPPORTED_PG_VERSION_OFFLINE) < 0 { return fmt.Errorf("current source db version: %s. Supported versions: %s", pgVersion, supportedVersionRange) } // for live migration if exportType == utils.CHANGES_ONLY || exportType == utils.SNAPSHOT_AND_CHANGES { - if version.CompareSimple(pgVersion, MIN_SUPPORTED_PG_VERSION_LIVE) < 0 { + if version.CompareSimple(majorVersion, MIN_SUPPORTED_PG_VERSION_LIVE) < 0 { supportedVersionRange = fmt.Sprintf("%s to %s", MIN_SUPPORTED_PG_VERSION_LIVE, MAX_SUPPORTED_PG_VERSION) utils.PrintAndLog(color.RedString("Warning: Live Migration: Current source db version: %s. Supported versions: %s", pgVersion, supportedVersionRange)) } From 521ac3147ab375e241c8c05fc60bbec0bd9f96c5 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 24 Dec 2024 16:59:58 +0530 Subject: [PATCH 074/105] [DB-14231] Report Security Invoker View in analyze and assess migration commands generated reports (#2108) - Implement ViewProcessor's `Process()` method using protomsg approach. - Made use of existing `TraverseParseTree()` function to write a helper func TraverseAndExtractDefNameFromDefElem()` to extract all defnames in any type of DDL. * Added unit tests * Added sql in end to end test for security invoker views --- migtests/scripts/functions.sh | 2 +- .../expectedAssessmentReport.json | 43 +++++++++++++---- .../tests/pg/assessment-report-test/init-db | 1 + .../pg_assessment_report.sql | 22 +++++++++ yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/cmd/assessMigrationCommand.go | 2 + yb-voyager/src/query/queryissue/constants.go | 3 ++ .../src/query/queryissue/detectors_ddl.go | 13 ++++- yb-voyager/src/query/queryissue/issues_ddl.go | 16 ++++++- .../src/query/queryissue/issues_ddl_test.go | 24 ++++++++++ .../queryissue/parser_issue_detector_test.go | 28 ++++++----- .../src/query/queryparser/ddl_processor.go | 32 +++++++++++-- .../src/query/queryparser/helpers_protomsg.go | 48 +++++++++++++++++++ .../src/query/queryparser/traversal_proto.go | 3 ++ 14 files changed, 209 insertions(+), 29 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index eeab236784..18f3ff9a28 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -932,7 +932,7 @@ compare_files() { return 0 else echo "Data does not match expected report." - diff_output=$(diff "$file1" "$file2") + diff_output=$(diff --context "$file1" "$file2") echo "$diff_output" return 1 fi diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index edfe82ecd2..1057ad9fd8 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 27, + "TotalCount": 28, "InvalidCount": 0, - "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees_employee_id_seq, public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 66, + "TotalCount": 67, "InvalidCount": 23, - "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" }, { "ObjectType": "INDEX", @@ -73,9 +73,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 7, - "InvalidCount": 3, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4" + "TotalCount": 8, + "InvalidCount": 4, + "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker" }, { "ObjectType": "TRIGGER", @@ -169,9 +169,10 @@ "test_views.abc_mview", "test_views.view_table1", "public.library_nested", - "public.orders_lateral" + "public.orders_lateral", + "public.employees" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 72 objects (64 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 73 objects (65 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -555,6 +556,16 @@ } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Security Invoker Views", + "Objects": [ + { + "ObjectName": "public.view_explicit_security_invoker", + "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employees.employee_id,\n employees.first_name\n FROM public.employees;" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -839,6 +850,20 @@ "ParentTableName": null, "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "employees", + "RowCount": 5, + "ColumnCount": 4, + "Reads": 0, + "Writes": 10, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, { "SchemaName": "public", "ObjectName": "employees2", diff --git a/migtests/tests/pg/assessment-report-test/init-db b/migtests/tests/pg/assessment-report-test/init-db index c4016a4c77..5160094487 100755 --- a/migtests/tests/pg/assessment-report-test/init-db +++ b/migtests/tests/pg/assessment-report-test/init-db @@ -25,6 +25,7 @@ cat < "$TEMP_SQL" \i ${TEST_DIR}/../views-and-rules/pg_views_and_rules_automation.sql; \i pg_assessment_report.sql; \i unsupported_query_constructs.sql; + CREATE SCHEMA IF NOT EXISTS schema2; SET SEARCH_PATH TO schema2; \i ${TEST_DIR}/../misc-objects-1/schema.sql; diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index 0c089922b5..c23e2ddc5f 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -362,3 +362,25 @@ BEGIN END IF; END; $$ LANGUAGE plpgsql; + +-- SECURITY INVOKER VIEW +CREATE TABLE public.employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(100), + last_name VARCHAR(100), + department VARCHAR(50) +); + +INSERT INTO public.employees (first_name, last_name, department) +VALUES + ('Alice', 'Smith', 'HR'), + ('Bob', 'Jones', 'Finance'), + ('Charlie', 'Brown', 'IT'), + ('Diana', 'Prince', 'HR'), + ('Ethan', 'Hunt', 'Security'); + +CREATE VIEW public.view_explicit_security_invoker +WITH (security_invoker = true) AS + SELECT employee_id, first_name + FROM public.employees; + diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 85d1ce25c7..a4dda2b03e 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -225,6 +225,7 @@ const ( INDEX_METHOD_ISSUE_REASON = "Schema contains %s index which is not supported." INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION = "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE" GIN_INDEX_DETAILS = "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." + SECURITY_INVOKER_VIEWS_ISSUE = "Security Invoker Views not supported yet" ) // Reports one case in JSON diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 0adcff8fe2..945c42e9ae 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -990,6 +990,7 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for PG...") unsupportedFeatures := make([]UnsupportedFeature, 0) + for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) featureName := fmt.Sprintf("%s indexes", displayIndexMethod) @@ -1022,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 3a6d424538..af2d24a6c0 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -53,6 +53,9 @@ const ( LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" + SECURITY_INVOKER_VIEWS = "SECURITY_INVOKER_VIEWS" + SECURITY_INVOKER_VIEWS_NAME = "Security Invoker Views" + ADVISORY_LOCKS = "ADVISORY_LOCKS" SYSTEM_COLUMNS = "SYSTEM_COLUMNS" XML_FUNCTIONS = "XML_FUNCTIONS" diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index e14da93bfa..532ee2dc25 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -545,7 +545,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer } if unsupportedLargeObjectFunctions.ContainsOne(trigger.FuncName) { - //Can't detect trigger func name using the genericIssues's FuncCallDetector + //Can't detect trigger func name using the genericIssues's FuncCallDetector //as trigger execute Func name is not a FuncCall node, its []pg_query.Node issues = append(issues, NewLOFuntionsIssue( obj.GetObjectType(), @@ -562,7 +562,16 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer type ViewIssueDetector struct{} func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { - return nil, nil + view, ok := obj.(*queryparser.View) + if !ok { + return nil, fmt.Errorf("invalid object type: expected View") + } + var issues []QueryIssue + + if view.SecurityInvoker { + issues = append(issues, NewSecurityInvokerViewIssue(obj.GetObjectType(), obj.GetObjectName(), "")) + } + return issues, nil } // ==============MVIEW ISSUE DETECTOR ====================== diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 65505d9d0d..914dcd5363 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -431,11 +431,23 @@ var loDatatypeIssue = issue.Issue{ TypeName: "Unsupported datatype - lo", Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", - DocsLink: "", //TODO + DocsLink: "", // TODO } func NewLODatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { issue := loDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) -} \ No newline at end of file +} + +var securityInvokerViewIssue = issue.Issue{ + Type: SECURITY_INVOKER_VIEWS, + TypeName: "Security Invoker Views not supported yet", + Suggestion: "Security Invoker Views are not yet supported in YugabyteDB, no workaround available currently", + GH: "", // TODO + DocsLink: "", // TODO +} + +func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(securityInvokerViewIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index d4d6fac326..68993779aa 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -216,6 +216,28 @@ func testLoDatatypeIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", loDatatypeIssue) } +func testSecurityInvokerView(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE public.employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(100), + last_name VARCHAR(100), + department VARCHAR(50) + ); + + CREATE VIEW public.view_explicit_security_invoker + WITH (security_invoker = true) AS + SELECT employee_id, first_name + FROM public.employees;`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "unrecognized parameter", securityInvokerViewIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -269,4 +291,6 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "lo datatype", ybVersion), testLoDatatypeIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "security invoker view", ybVersion), testSecurityInvokerView) + assert.True(t, success) } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e7c979118f..066d738fb2 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -156,9 +156,13 @@ ADD CONSTRAINT valid_invoice_structure CHECK (xpath_exists('/invoice/customer', data));` stmt18 = `CREATE INDEX idx_invoices on invoices (xpath('/invoice/customer/text()', data));` stmt19 = `create table test_lo_default (id int, raster lo DEFAULT lo_import('3242'));` + stmt20 = `CREATE VIEW public.view_explicit_security_invoker + WITH (security_invoker = true) AS + SELECT employee_id, first_name + FROM public.employees;` ) -func modifyiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { +func modifiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { i.ObjectType = objType i.ObjectName = objName @@ -221,9 +225,9 @@ func TestAllIssues(t *testing.T) { } //Should modify it in similar way we do it actual code as the particular DDL issue in plpgsql can have different Details map on the basis of objectType - stmtsWithExpectedIssues[stmt1] = modifyiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt1], "FUNCTION", "list_high_earners") + stmtsWithExpectedIssues[stmt1] = modifiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt1], "FUNCTION", "list_high_earners") - stmtsWithExpectedIssues[stmt2] = modifyiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt2], "FUNCTION", "process_order") + stmtsWithExpectedIssues[stmt2] = modifiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt2], "FUNCTION", "process_order") for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -276,6 +280,9 @@ func TestDDLIssues(t *testing.T) { NewLODatatypeIssue("TABLE", "test_lo_default", stmt19, "raster"), NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19), }, + stmt20: []QueryIssue{ + NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), + }, } for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -415,7 +422,7 @@ BEGIN END; $$ LANGUAGE plpgsql; `, -`CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON image + `CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON image FOR EACH ROW EXECUTE FUNCTION lo_manage(raster);`, } @@ -442,12 +449,11 @@ $$ LANGUAGE plpgsql; NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5]), }, } - expectedSQLsWithIssues[sqls[0]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") - expectedSQLsWithIssues[sqls[1]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") - expectedSQLsWithIssues[sqls[2]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") - expectedSQLsWithIssues[sqls[3]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") - expectedSQLsWithIssues[sqls[4]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") - + expectedSQLsWithIssues[sqls[0]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") + expectedSQLsWithIssues[sqls[1]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") + expectedSQLsWithIssues[sqls[2]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") + expectedSQLsWithIssues[sqls[3]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") + expectedSQLsWithIssues[sqls[4]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") parserIssueDetector := NewParserIssueDetector() @@ -456,7 +462,6 @@ $$ LANGUAGE plpgsql; fmt.Printf("%v", issues) assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) - assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { @@ -466,6 +471,7 @@ $$ LANGUAGE plpgsql; } } } + // currently, both FuncCallDetector and XmlExprDetector can detect XMLFunctionsIssue // statement below has both XML functions and XML expressions. // but we want to only return one XMLFunctionsIssue from parserIssueDetector.getDMLIssues diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 2d2a6e7665..a14485cd89 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -22,6 +22,7 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" + log "github.com/sirupsen/logrus" ) // Base parser interface @@ -849,16 +850,39 @@ func (v *ViewProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, err if !ok { return nil, fmt.Errorf("not a CREATE VIEW statement") } + + viewSchemaName := viewNode.ViewStmt.View.Schemaname + viewName := viewNode.ViewStmt.View.Relname + qualifiedViewName := lo.Ternary(viewSchemaName == "", viewName, viewSchemaName+"."+viewName) + + /* + view_stmt:{view:{schemaname:"public" relname:"invoker_view" inh:true relpersistence:"p" location:12} + query:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{string:{sval:"id"}} location:95}} location:95}} + from_clause:{...} + where_clause:{...} + options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} + options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} + with_check_option:NO_CHECK_OPTION} + */ + log.Infof("checking the view '%s' is security invoker view", qualifiedViewName) + msg := GetProtoMessageFromParseTree(parseTree) + defNames, err := TraverseAndExtractDefNamesFromDefElem(msg) + if err != nil { + return nil, err + } + view := View{ - SchemaName: viewNode.ViewStmt.View.Schemaname, - ViewName: viewNode.ViewStmt.View.Relname, + SchemaName: viewSchemaName, + ViewName: viewName, + SecurityInvoker: slices.Contains(defNames, "security_invoker"), } return &view, nil } type View struct { - SchemaName string - ViewName string + SchemaName string + ViewName string + SecurityInvoker bool } func (v *View) GetObjectName() string { diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 0eb609d15f..c79aebc8c1 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -16,6 +16,7 @@ limitations under the License. package queryparser import ( + "fmt" "strings" pg_query "github.com/pganalyze/pg_query_go/v6" @@ -328,6 +329,21 @@ func GetStatementType(msg protoreflect.Message) string { return GetMsgFullName(node) } +func getOneofActiveNode(msg protoreflect.Message) protoreflect.Message { + nodeField := getOneofActiveField(msg, "node") + if nodeField == nil { + return nil + } + + value := msg.Get(nodeField) + node := value.Message() + if node == nil || !node.IsValid() { + return nil + } + + return node +} + // == Generic helper functions == // GetStringField retrieves a string field from a message. @@ -370,3 +386,35 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { } return schemaName, objectName } + +/* +Example: +options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} +options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} + +Extract all defnames from the def_eleme node +*/ +func TraverseAndExtractDefNamesFromDefElem(msg protoreflect.Message) ([]string, error) { + var defNames []string + + collectorFunc := func(msg protoreflect.Message) error { + if GetMsgFullName(msg) != PG_QUERY_DEFELEM_NODE { + return nil + } + + defName := GetStringField(msg, "defname") + // TODO(future): + // defValNode = GetMessageField(msg, "arg") + // defVal = GetStringField(defValNode, "sval") + + defNames = append(defNames, defName) + return nil + } + visited := make(map[protoreflect.Message]bool) + err := TraverseParseTree(msg, visited, collectorFunc) + if err != nil { + return nil, fmt.Errorf("failed to traverse parse tree for fetching defnames: %w", err) + } + + return defNames, nil +} diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 3ed84f4ab5..d54aefe0ef 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -36,10 +36,13 @@ const ( PG_QUERY_RANGEVAR_NODE = "pg_query.RangeVar" PG_QUERY_RANGETABLEFUNC_NODE = "pg_query.RangeTableFunc" PG_QUERY_PARAMREF_NODE = "pg_query.ParamRef" + PG_QUERY_DEFELEM_NODE = "pg_query.DefElem" PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" + + PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" ) // function type for processing nodes during traversal From 89c4f76a52ec838e74e7c1ae6df7de70e076dff0 Mon Sep 17 00:00:00 2001 From: Shaharuk Shaikh <56402576+shaharuk-yb@users.noreply.github.com> Date: Tue, 24 Dec 2024 18:35:47 +0530 Subject: [PATCH 075/105] [Sizing Calc] Consider specific object types, skip considering primary key object type in sizing calculator workflow (#2089) * skip considering primary key object type in sizing calculator workflow --- yb-voyager/src/migassessment/sizing.go | 18 +++++++++++++++++- yb-voyager/src/migassessment/sizing_test.go | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/migassessment/sizing.go b/yb-voyager/src/migassessment/sizing.go index 1d45a1167b..05383341a1 100644 --- a/yb-voyager/src/migassessment/sizing.go +++ b/yb-voyager/src/migassessment/sizing.go @@ -25,6 +25,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "github.com/samber/lo" @@ -139,6 +140,12 @@ func getExperimentDBPath() string { //go:embed resources/yb_2024_0_source.db var experimentData20240 []byte +var SourceMetadataObjectTypesToUse = []string{ + "%table%", + "%index%", + "materialized view", +} + func SizingAssessment() error { log.Infof("loading metadata files for sharding assessment") @@ -1228,6 +1235,14 @@ Returns: float64: The total size of the source database in gigabytes. */ func getSourceMetadata(sourceDB *sql.DB) ([]SourceDBMetadata, []SourceDBMetadata, float64, error) { + // Construct the WHERE clause dynamically using LIKE + var likeConditions []string + for _, pattern := range SourceMetadataObjectTypesToUse { + likeConditions = append(likeConditions, fmt.Sprintf("object_type LIKE '%s'", pattern)) + } + // Join the LIKE conditions with OR + whereClause := strings.Join(likeConditions, " OR ") + query := fmt.Sprintf(` SELECT schema_name, object_name, @@ -1239,8 +1254,9 @@ func getSourceMetadata(sourceDB *sql.DB) ([]SourceDBMetadata, []SourceDBMetadata size_in_bytes, column_count FROM %v + WHERE %s ORDER BY IFNULL(size_in_bytes, 0) ASC - `, GetTableIndexStatName()) + `, GetTableIndexStatName(), whereClause) rows, err := sourceDB.Query(query) if err != nil { return nil, nil, 0.0, fmt.Errorf("failed to query source metadata with query [%s]: %w", query, err) diff --git a/yb-voyager/src/migassessment/sizing_test.go b/yb-voyager/src/migassessment/sizing_test.go index bdf28050fc..dff6c52522 100644 --- a/yb-voyager/src/migassessment/sizing_test.go +++ b/yb-voyager/src/migassessment/sizing_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/assert" ) -var AssessmentDbSelectQuery = fmt.Sprintf("(?i)SELECT schema_name,.* FROM %v ORDER BY .* ASC", TABLE_INDEX_STATS) +var AssessmentDbSelectQuery = fmt.Sprintf("(?i)SELECT schema_name,.* FROM %v .* ORDER BY .* ASC", TABLE_INDEX_STATS) var AssessmentDBColumns = []string{"schema_name", "object_name", "row_count", "reads_per_second", "writes_per_second", "is_index", "parent_table_name", "size_in_bytes", "column_count"} From 3c45ffc29061afd0d996ddbcbbd07bef5b734d14 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Tue, 24 Dec 2024 18:47:51 +0530 Subject: [PATCH 076/105] Reporting Json Constructor / Query Functions and AnyValue Aggregate functions (#2107) 1. Added support to report the JSON constructor functions (`JSON_OBJECT`, `JSON_OBJECTAGG`, `JSON_ARRAYAGG`, `JSON_ARRAY`) and JSON query functions (`JSON_QUERY`, `JSON_EXISTS`, `JSON_VALUE`) with new detectors as these all have a separate to detect. Currently, the code and unit tests are added for reporting the `JSON_TABLE` in DMLs but it can't be reported in the Query constructs sections as PGSS is giving the queries with placeholders ($1, $2...) and `JSON_TABLE` syntax doesn't allow it so the parser is failing with that. https://yugabyte.atlassian.net/browse/DB-14542 2. Added support to report the aggregate functions, reporting the `ANY_VALUE` agg function in this PR. https://yugabyte.atlassian.net/browse/DB-14220 3. Fixed the assessment template for adding the Supported Versions in HTML for PL/pgSQL and Migration Caveats sections. --- .../dummy-export-dir/schema/views/view.sql | 13 +- .../tests/analyze-schema/expected_issues.json | 30 ++++ migtests/tests/analyze-schema/summary.json | 6 +- .../pg_assessment_report_uqc.sql | 6 +- .../unsupported_query_constructs.sql | 7 +- yb-voyager/cmd/assessMigrationCommand.go | 3 + .../migration_assessment_report.template | 22 ++- yb-voyager/src/query/queryissue/constants.go | 13 +- yb-voyager/src/query/queryissue/detectors.go | 112 +++++++++++++- .../src/query/queryissue/detectors_ddl.go | 1 + .../src/query/queryissue/detectors_test.go | 123 +++++++-------- yb-voyager/src/query/queryissue/helpers.go | 18 +++ yb-voyager/src/query/queryissue/issues_dml.go | 65 +++++++- .../src/query/queryissue/issues_dml_test.go | 63 +++++++- .../query/queryissue/parser_issue_detector.go | 2 + .../queryissue/parser_issue_detector_test.go | 146 ++++++++++++++++-- .../src/query/queryparser/helpers_protomsg.go | 12 ++ .../src/query/queryparser/traversal_proto.go | 8 +- 18 files changed, 560 insertions(+), 90 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index 435adb925f..157d311ceb 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -32,4 +32,15 @@ CREATE VIEW public.orders_view AS pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id - FROM public.orders; \ No newline at end of file + FROM public.orders; + +CREATE VIEW public.my_films_view AS +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' + COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text FORMAT JSON PATH '$.title' OMIT QUOTES, + director text PATH '$.director' KEEP QUOTES))) AS jt; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 35cc21ec22..28e11bd8c4 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -20,6 +20,36 @@ "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "public.my_films_view", + "Reason": "Json Query Functions", + "SqlStatement": "CREATE VIEW public.my_films_view AS\nSELECT jt.* FROM\n my_films,\n JSON_TABLE ( js, '$.favorites[*]'\n COLUMNS (\n id FOR ORDINALITY,\n kind text PATH '$.kind',\n NESTED PATH '$.films[*]' COLUMNS (\n title text FORMAT JSON PATH '$.title' OMIT QUOTES,\n director text PATH '$.director' KEEP QUOTES))) AS jt;", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "test", + "Reason": "Json Constructor Functions", + "SqlStatement": "CREATE OR REPLACE view test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "MVIEW", + "ObjectName": "test", + "Reason": "Json Constructor Functions", + "SqlStatement": "CREATE MATERIALIZED VIEW test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 969e87da89..f2d1304f39 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -50,9 +50,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 5, - "InvalidCount": 5, - "ObjectNames": "v1, v2, test, public.orders_view, view_name" + "TotalCount": 6, + "InvalidCount": 6, + "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name" }, { "ObjectType": "TRIGGER", diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index c4eff19e41..2b42d33348 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -41,4 +41,8 @@ INSERT INTO hr.departments (department_name, location) VALUES ('Engineering', 'B INSERT INTO hr.departments (department_name, location) VALUES ('Sales', 'Building B'); INSERT INTO public.employees (name, department_id) VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 2); INSERT INTO sales.orders (customer_id, amount) VALUES (101, 500.00), (102, 1200.00); -INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05); \ No newline at end of file +INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05); + +create view sales.employ_depart_view AS SELECT + any_value(name) AS any_employee + FROM employees; \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index a0ebe3d97a..1840c36242 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -20,4 +20,9 @@ FROM public.employees; -- 4) Advisory locks (analytics schema) SELECT metric_name, pg_advisory_lock(metric_id) FROM analytics.metrics -WHERE metric_value > 0.02; \ No newline at end of file +WHERE metric_value > 0.02; + +-- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15 +SELECT + any_value(name) AS any_employee + FROM employees; \ No newline at end of file diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 945c42e9ae..12cae929bd 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1023,6 +1023,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 6681f0eaee..6fe380e1e5 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -286,7 +286,7 @@ {{ if $docsLink }} Docs Link {{ else }} - Not Available + N/A {{ end }}     @@ -313,6 +313,7 @@ {{ $objectsGroupByObjectType := groupByObjectType .Objects }} {{ $numUniqueObjectNamesOfAllTypes := totalUniqueObjectNamesOfAllTypes $objectsGroupByObjectType }} {{ $docsLink := .DocsLink }} + {{ $supportedVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} {{ $isNextRowRequiredForObjectType := false }} {{ range $type, $objectsByType := $objectsGroupByObjectType }} @@ -338,7 +339,16 @@ {{ if not $isNextRowRequiredForObjectType }} - + {{ end }} {{ $isNextRowRequiredForObjectName = true }} {{ $isNextRowRequiredForObjectType = true }} @@ -359,6 +369,10 @@ {{ $hasMigrationCaveats = true }} {{if .DisplayDDL }}

    {{.FeatureName}}

    + {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +

    Supported in Versions: {{ $supporterVerStr }}

    + {{ end }}

    {{.FeatureDescription}}

      @@ -369,6 +383,10 @@
    {{else}}

    {{.FeatureName}}

    + {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +

    Supported in Versions: {{ $supporterVerStr }}

    + {{ end }}

    {{.FeatureDescription}}

      diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index af2d24a6c0..f0f86239a5 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -49,9 +49,15 @@ const ( FOREIGN_TABLE = "FOREIGN_TABLE" INHERITANCE = "INHERITANCE" - LARGE_OBJECT_DATATYPE = "LARGE_OBJECT_DATATYPE" - LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" - LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" + AGGREGATE_FUNCTION = "AGGREGATE_FUNCTION" + AGGREGATION_FUNCTIONS_NAME = "Aggregate Functions" + JSON_CONSTRUCTOR_FUNCTION = "JSON_CONSTRUCTOR_FUNCTION" + JSON_CONSTRUCTOR_FUNCTION_NAME = "Json Constructor Functions" + JSON_QUERY_FUNCTION = "JSON_QUERY_FUNCTION" + JSON_QUERY_FUNCTIONS_NAME = "Json Query Functions" + LARGE_OBJECT_DATATYPE = "LARGE_OBJECT_DATATYPE" + LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" + LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" SECURITY_INVOKER_VIEWS = "SECURITY_INVOKER_VIEWS" SECURITY_INVOKER_VIEWS_NAME = "Security Invoker Views" @@ -69,6 +75,7 @@ const ( // Object types const ( CONSTRAINT_NAME = "ConstraintName" + FUNCTION_NAMES = "FunctionNames" TABLE_OBJECT_TYPE = "TABLE" FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" FUNCTION_OBJECT_TYPE = "FUNCTION" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index bb9490786f..8a57c581df 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -35,6 +35,7 @@ type FuncCallDetector struct { advisoryLocksFuncsDetected mapset.Set[string] xmlFuncsDetected mapset.Set[string] + aggFuncsDetected mapset.Set[string] regexFuncsDetected mapset.Set[string] loFuncsDetected mapset.Set[string] } @@ -44,6 +45,7 @@ func NewFuncCallDetector(query string) *FuncCallDetector { query: query, advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](), xmlFuncsDetected: mapset.NewThreadUnsafeSet[string](), + aggFuncsDetected: mapset.NewThreadUnsafeSet[string](), regexFuncsDetected: mapset.NewThreadUnsafeSet[string](), loFuncsDetected: mapset.NewThreadUnsafeSet[string](), } @@ -68,6 +70,9 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) error { d.regexFuncsDetected.Add(funcName) } + if unsupportedAggFunctions.ContainsOne(funcName) { + d.aggFuncsDetected.Add(funcName) + } if unsupportedLargeObjectFunctions.ContainsOne(funcName) { d.loFuncsDetected.Add(funcName) } @@ -83,11 +88,14 @@ func (d *FuncCallDetector) GetIssues() []QueryIssue { if d.xmlFuncsDetected.Cardinality() > 0 { issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } + if d.aggFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.aggFuncsDetected.ToSlice())) + } if d.regexFuncsDetected.Cardinality() > 0 { issues = append(issues, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } if d.loFuncsDetected.Cardinality() > 0 { - issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.loFuncsDetected.ToSlice())) } return issues } @@ -195,3 +203,105 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { } return issues } + +type JsonConstructorFuncDetector struct { + query string + unsupportedJsonConstructorFunctionsDetected mapset.Set[string] +} + +func NewJsonConstructorFuncDetector(query string) *JsonConstructorFuncDetector { + return &JsonConstructorFuncDetector{ + query: query, + unsupportedJsonConstructorFunctionsDetected: mapset.NewThreadUnsafeSet[string](), + } +} + +func (j *JsonConstructorFuncDetector) Detect(msg protoreflect.Message) error { + switch queryparser.GetMsgFullName(msg) { + case queryparser.PG_QUERY_JSON_ARRAY_AGG_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_ARRAYAGG) + case queryparser.PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_ARRAY) + case queryparser.PG_QUERY_JSON_OBJECT_AGG_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_OBJECTAGG) + case queryparser.PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_OBJECT) + } + return nil +} + +func (d *JsonConstructorFuncDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.unsupportedJsonConstructorFunctionsDetected.Cardinality() > 0 { + issues = append(issues, NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.unsupportedJsonConstructorFunctionsDetected.ToSlice())) + } + return issues +} + +type JsonQueryFunctionDetector struct { + query string + unsupportedJsonQueryFunctionsDetected mapset.Set[string] +} + +func NewJsonQueryFunctionDetector(query string) *JsonQueryFunctionDetector { + return &JsonQueryFunctionDetector{ + query: query, + unsupportedJsonQueryFunctionsDetected: mapset.NewThreadUnsafeSet[string](), + } +} + +func (j *JsonQueryFunctionDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_TABLE_NODE { + /* + SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + ); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} + from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} + location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} + columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} + pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... + */ + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) + return nil + } + if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_JSON_FUNC_EXPR_NODE { + return nil + } + /* + JsonExprOp - + enumeration of SQL/JSON query function types + typedef enum JsonExprOp + { + 1. JSON_EXISTS_OP, JSON_EXISTS() + 2. JSON_QUERY_OP, JSON_QUERY() + 3. JSON_VALUE_OP, JSON_VALUE() + 4. JSON_TABLE_OP, JSON_TABLE() + } JsonExprOp; + */ + jsonExprFuncOpNum := queryparser.GetEnumNumField(msg, "op") + switch jsonExprFuncOpNum { + case 1: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_EXISTS) + case 2: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_QUERY) + case 3: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_VALUE) + case 4: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) + } + return nil +} + +func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.unsupportedJsonQueryFunctionsDetected.Cardinality() > 0 { + issues = append(issues, NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.unsupportedJsonQueryFunctionsDetected.ToSlice())) + } + return issues +} diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 532ee2dc25..82ea56d048 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -551,6 +551,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer obj.GetObjectType(), trigger.GetObjectName(), "", + []string{trigger.FuncName}, )) } diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 48b8d674c5..266f053ba0 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -27,6 +27,26 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) +func getDetectorIssues(t *testing.T, detector UnsupportedConstructDetector, sql string) []QueryIssue { + parseResult, err := queryparser.Parse(sql) + assert.NoError(t, err, "Failed to parse SQL: %s", sql) + + visited := make(map[protoreflect.Message]bool) + + processor := func(msg protoreflect.Message) error { + err := detector.Detect(msg) + if err != nil { + return err + } + return nil + } + + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) + err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + return detector.GetIssues() +} + func TestFuncCallDetector(t *testing.T) { advisoryLockSqls := []string{ `SELECT pg_advisory_lock(100), COUNT(*) FROM cars;`, @@ -76,6 +96,13 @@ func TestFuncCallDetector(t *testing.T) { `SELECT pg_advisory_unlock_all();`, } + anyValAggSqls := []string{ + `SELECT + department, + any_value(employee_name) AS any_employee + FROM employees + GROUP BY department;`, + } loFunctionSqls := []string{ `UPDATE documents SET content_oid = lo_import('/path/to/new/file.pdf') @@ -90,39 +117,23 @@ WHERE title = 'Design Document';`, `SELECT lo_unlink((SELECT content_oid FROM documents WHERE title = 'Sample Document'));`, `create table test_lo_default (id int, raster lo DEFAULT lo_import('3242'));`, } - - detectConstructs := func(sql string) []QueryIssue { - detector := NewFuncCallDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - return detector.GetIssues() - } - for _, sql := range advisoryLockSqls { - issues := detectConstructs(sql) - assert.Equal(t, len(issues), 1) - assert.Equal(t, issues[0].Type, ADVISORY_LOCKS, "Advisory Locks not detected in SQL: %s", sql) + + issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, ADVISORY_LOCKS, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } for _, sql := range loFunctionSqls { - issues := detectConstructs(sql) + issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) assert.Equal(t, len(issues), 1) assert.Equal(t, issues[0].Type, LARGE_OBJECT_FUNCTIONS, "Large Objects not detected in SQL: %s", sql) + } + for _, sql := range anyValAggSqls { + issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, AGGREGATE_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } } @@ -168,25 +179,7 @@ func TestColumnRefDetector(t *testing.T) { } for _, sql := range systemColumnSqls { - detector := NewColumnRefDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - - issues := detector.GetIssues() + issues := getDetectorIssues(t, NewColumnRefDetector(sql), sql) assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) assert.Equal(t, SYSTEM_COLUMNS, issues[0].Type, "Expected System Columns issue for SQL: %s", sql) @@ -354,25 +347,7 @@ func TestRangeTableFuncDetector(t *testing.T) { } for _, sql := range xmlTableSqls { - detector := NewRangeTableFuncDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - - issues := detector.GetIssues() + issues := getDetectorIssues(t, NewRangeTableFuncDetector(sql), sql) assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) assert.Equal(t, XML_FUNCTIONS, issues[0].Type, "Expected XML Functions issue for SQL: %s", sql) @@ -713,3 +688,23 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { "Schema list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedSchemas, len(tc.ExpectedSchemas), collectedSchemas, len(collectedSchemas)) } } + +func TestJsonConstructorDetector(t *testing.T) { + sql := `SELECT JSON_ARRAY('PostgreSQL', 12, TRUE, NULL) AS json_array;` + + issues := getDetectorIssues(t, NewJsonConstructorFuncDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSON_CONSTRUCTOR_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + +} + +func TestJsonQueryFunctionDetector(t *testing.T) { + sql := `SELECT id, JSON_VALUE(details, '$.title') AS title +FROM books +WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);` + + issues := getDetectorIssues(t, NewJsonQueryFunctionDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSON_QUERY_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + +} diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index 617b2fb701..d24a14ed53 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -102,6 +102,24 @@ var UnsupportedIndexDatatypes = []string{ // array as well but no need to add it in the list as fetching this type is a different way TODO: handle better with specific types } +var unsupportedAggFunctions = mapset.NewThreadUnsafeSet([]string{ + //agg function added in PG16 - https://www.postgresql.org/docs/16/functions-aggregate.html#id-1.5.8.27.5.2.4.1.1.1.1 + "any_value", +}...) + +const ( + // // json functions, refer - https://www.postgresql.org/about/featurematrix/detail/395/ + JSON_OBJECTAGG = "JSON_OBJECTAGG" + JSON_ARRAY = "JSON_ARRAY" + JSON_ARRAYAGG = "JSON_ARRAYAGG" + JSON_OBJECT = "JSON_OBJECT" + //json query functions supported in PG 17, refer - https://www.postgresql.org/docs/17/functions-json.html#FUNCTIONS-SQLJSON-QUERYING + JSON_EXISTS = "JSON_EXISTS" + JSON_QUERY = "JSON_QUERY" + JSON_VALUE = "JSON_VALUE" + JSON_TABLE = "JSON_TABLE" +) + var unsupportedLargeObjectFunctions = mapset.NewThreadUnsafeSet([]string{ //refer - https://www.postgresql.org/docs/current/lo-interfaces.html#LO-CREATE diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 33db98be0f..43dc8ab07f 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -16,7 +16,11 @@ limitations under the License. package queryissue -import "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" +import ( + "sort" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" +) var advisoryLocksIssue = issue.Issue{ Type: ADVISORY_LOCKS, @@ -70,6 +74,57 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s return newQueryIssue(regexFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } +var anyValueAggFunctionIssue = issue.Issue{ + Type: AGGREGATE_FUNCTION, + TypeName: AGGREGATION_FUNCTIONS_NAME, + TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(anyValueAggFunctionIssue, objectType, objectName, sqlStatement, details) +} + +var jsonConstructorFunctionsIssue = issue.Issue{ + Type: JSON_CONSTRUCTOR_FUNCTION, + TypeName: JSON_CONSTRUCTOR_FUNCTION_NAME, + TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(jsonConstructorFunctionsIssue, objectType, objectName, sqlStatement, details) +} + +var jsonQueryFunctionIssue = issue.Issue{ + Type: JSON_QUERY_FUNCTION, + TypeName: JSON_QUERY_FUNCTIONS_NAME, + TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(jsonQueryFunctionIssue, objectType, objectName, sqlStatement, details) +} + var loFunctionsIssue = issue.Issue{ Type: LARGE_OBJECT_FUNCTIONS, TypeName: LARGE_OBJECT_FUNCTIONS_NAME, @@ -79,6 +134,10 @@ var loFunctionsIssue = issue.Issue{ DocsLink: "", //TODO } -func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { - return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index af3d8afcc1..5b0cd0151b 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -46,7 +46,6 @@ func testRegexFunctionsIssue(t *testing.T) { assert.NoError(t, err) defer conn.Close(context.Background()) - stmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, `SELECT regexp_instr('This is an example. Another example. Example is a common word.', 'example')`, @@ -59,6 +58,62 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testJsonConstructorFunctions(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + sqls := map[string]string{ + `select json_object('code' VALUE 'P123', 'title': 'Jaws');`: `syntax error at or near "VALUE"`, + `select JSON_ARRAYAGG('[1, "2", null]');`: `does not exist`, + `SELECT json_objectagg(k VALUE v) AS json_result + FROM (VALUES ('a', 1), ('b', 2), ('c', 3)) AS t(k, v);`: `syntax error at or near "VALUE"`, + `SELECT JSON_ARRAY('PostgreSQL', 12, TRUE, NULL) AS json_array;`: `does not exist`, + } + for sql, expectedErr := range sqls { + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, expectedErr, jsonConstructorFunctionsIssue) + } +} + +func testJsonQueryFunctions(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + sqls := []string{ + `SELECT id, JSON_QUERY(details, '$.author') AS author +FROM books;`, + `SELECT + id, + JSON_VALUE(details, '$.title') AS title, + JSON_VALUE(details, '$.price')::NUMERIC AS price +FROM books;`, + `SELECT id, details +FROM books +WHERE JSON_EXISTS(details, '$.author');`, + } + for _, sql := range sqls { + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `does not exist`, jsonConstructorFunctionsIssue) + } + + jsonTableSQL := `SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + );` + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, jsonTableSQL) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "COLUMNS"`, jsonConstructorFunctionsIssue) +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -90,4 +145,10 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "json constructor functions", ybVersion), testJsonConstructorFunctions) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "json query functions", ybVersion), testJsonQueryFunctions) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index ffa1d98605..8b5f028b95 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,8 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewJsonConstructorFuncDetector(query), + NewJsonQueryFunctionDetector(query), } processor := func(msg protoreflect.Message) error { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 066d738fb2..1450f4edd8 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -278,7 +278,7 @@ func TestDDLIssues(t *testing.T) { }, stmt19: []QueryIssue{ NewLODatatypeIssue("TABLE", "test_lo_default", stmt19, "raster"), - NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19), + NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19, []string{"lo_import"}), }, stmt20: []QueryIssue{ NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), @@ -428,25 +428,25 @@ $$ LANGUAGE plpgsql; expectedSQLsWithIssues := map[string][]QueryIssue{ sqls[0]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_unlink(loid);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_unlink(loid);", []string{"lo_unlink"}), }, sqls[1]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "INSERT INTO documents (title, content_oid) VALUES (doc_title, lo_import(file_path));"), + NewLOFuntionsIssue("DML_QUERY", "", "INSERT INTO documents (title, content_oid) VALUES (doc_title, lo_import(file_path));", []string{"lo_import"}), }, sqls[2]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_export(loid, file_path);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_export(loid, file_path);", []string{"lo_export"}), }, sqls[3]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);", []string{"lo_close"}), }, sqls[4]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_put(fd, convert_to(new_data, 'UTF8'));"), - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_put(fd, convert_to(new_data, 'UTF8'));", []string{"lo_put"}), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);", []string{"lo_close"}), NewLODatatypeIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));", "raster"), - NewLOFuntionsIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));"), + NewLOFuntionsIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));", []string{"lo_import"}), }, sqls[5]: []QueryIssue{ - NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5]), + NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5], []string{"lo_manage"}), }, } expectedSQLsWithIssues[sqls[0]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") @@ -492,6 +492,134 @@ func TestSingleXMLIssueIsDetected(t *testing.T) { assert.Equal(t, 1, len(issues)) } +func TestJsonUnsupportedFeatures(t *testing.T) { + sqls := []string{ + `SELECT department, JSON_ARRAYAGG(name) AS employees_json + FROM employees + GROUP BY department;`, + `INSERT INTO movies (details) +VALUES ( + JSON_OBJECT('title' VALUE 'Dune', 'director' VALUE 'Denis Villeneuve', 'year' VALUE 2021) +);`, + `SELECT json_objectagg(k VALUE v) AS json_result + FROM (VALUES ('a', 1), ('b', 2), ('c', 3)) AS t(k, v);`, + `SELECT JSON_OBJECT( + 'movie' VALUE JSON_OBJECT('code' VALUE 'P123', 'title' VALUE 'Jaws'), + 'director' VALUE 'Steven Spielberg' +) AS nested_json_object;`, + `select JSON_ARRAYAGG('[1, "2", null]');`, + `SELECT JSON_OBJECT( + 'code' VALUE 'P123', + 'title' VALUE 'Jaws', + 'price' VALUE 19.99, + 'available' VALUE TRUE +) AS json_obj;`, + `SELECT id, JSON_QUERY(details, '$.author') AS author +FROM books;`, + `SELECT jt.* FROM + my_films, + JSON_TABLE (js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + title text PATH '$.films[*].title' WITH WRAPPER, + director text PATH '$.films[*].director' WITH WRAPPER)) AS jt;`, + `SELECT jt.* FROM + my_films, + JSON_TABLE (js, $1 COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + title text PATH '$.films[*].title' WITH WRAPPER, + director text PATH '$.films[*].director' WITH WRAPPER)) AS jt;`, + `SELECT id, details +FROM books +WHERE JSON_EXISTS(details, '$.author');`, + `SELECT id, JSON_QUERY(details, '$.author') AS author +FROM books;`, + `SELECT + id, + JSON_VALUE(details, '$.title') AS title, + JSON_VALUE(details, '$.price')::NUMERIC AS price +FROM books;`, + `SELECT id, JSON_VALUE(details, '$.title') AS title +FROM books +WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`, +`CREATE MATERIALIZED VIEW public.test_jsonb_view AS +SELECT + id, + data->>'name' AS name, + JSON_VALUE(data, '$.age' RETURNING INTEGER) AS age, + JSON_EXISTS(data, '$.skills[*] ? (@ == "JSON")') AS knows_json, + jt.skill +FROM public.test_jsonb, +JSON_TABLE(data, '$.skills[*]' + COLUMNS ( + skill TEXT PATH '$' + ) +) AS jt;`, + `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, + } + sqlsWithExpectedIssues := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[0], []string{JSON_ARRAYAGG}), + }, + sqls[1]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[1], []string{JSON_OBJECT}), + }, + sqls[2]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[2], []string{JSON_OBJECTAGG}), + }, + sqls[3]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[3], []string{JSON_OBJECT}), + }, + sqls[4]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[4], []string{JSON_ARRAYAGG}), + }, + sqls[5]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[5], []string{JSON_OBJECT}), + }, + sqls[6]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[6], []string{JSON_QUERY}), + }, + sqls[7]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[7], []string{JSON_TABLE}), + }, + // sqls[8]: []QueryIssue{ + // NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[8]), + //NOT REPORTED YET because of PARSER failing if JSON_TABLE has a parameterized values $1, $2 ... + //https://github.com/pganalyze/pg_query_go/issues/127 + // }, + sqls[9]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[9], []string{JSON_EXISTS}), + }, + sqls[10]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[10], []string{JSON_QUERY}), + }, + sqls[11]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[11], []string{JSON_VALUE}), + }, + sqls[12]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[12], []string{JSON_VALUE, JSON_EXISTS}), + }, + sqls[13]: []QueryIssue{ + NewJsonQueryFunctionIssue("MVIEW", "public.test_jsonb_view", sqls[13], []string{JSON_VALUE, JSON_EXISTS, JSON_TABLE}), + }, + sqls[14]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[14], []string{JSON_ARRAY}), + }, + } + parserIssueDetector := NewParserIssueDetector() + for stmt, expectedIssues := range sqlsWithExpectedIssues { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} func TestRegexFunctionsIssue(t *testing.T) { dmlStmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index c79aebc8c1..e0de006047 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -34,6 +34,7 @@ const ( func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { return parseTree.Stmts[0].Stmt.ProtoReflect() + } func GetMsgFullName(msg protoreflect.Message) string { @@ -374,6 +375,17 @@ func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List return nil } +// GetEnumNumField retrieves a enum field from a message +// FieldDescriptor{Syntax: proto3, FullName: pg_query.JsonFuncExpr.op, Number: 1, Cardinality: optional, Kind: enum, HasJSONName: true, JSONName: "op", Enum: pg_query.JsonExprOp} +//val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT +func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.EnumNumber { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).Enum() + } + return 0 +} + // GetSchemaAndObjectName extracts the schema and object name from a list. func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { var schemaName, objectName string diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index d54aefe0ef..c988ee48eb 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -42,7 +42,13 @@ const ( PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" - PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" + PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" + PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" + PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE = "pg_query.JsonArrayConstructor" + PG_QUERY_JSON_FUNC_EXPR_NODE = "pg_query.JsonFuncExpr" + PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" + PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" + PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" ) // function type for processing nodes during traversal From 43aaeeb74d26290fa154995c694a5b25acfed589 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 24 Dec 2024 19:50:56 +0530 Subject: [PATCH 077/105] Introduce TestContainer interface to streamline test setup of supported databases (#2053) - Any package in voyager code, for eg srcdb, tgtdb, connpool, yugabyted etc.. can spin up the required source db(oracle, pg, mysql) with a specific version to write unit tests. - Every call to TestContainer.Start() check if a required database(dbtype + version) is already run then reuse it, otherwise start a new one - In every package add a TestMain() function, to setup the environment needed specifically for that test. - Added a new Github actions workflow to run the integration tests separately(required some oracle instance client libraries). - Moving forward it will be good to have separation between go tests as unit or integration * Bug fix: GetNonPKTables() for PG and YB returned non table objects like sequences and pk constraint objects - fixed to return only table names --- .github/workflows/go.yml | 4 +- .github/workflows/integration-tests.yml | 53 ++++++ .github/workflows/issue-tests.yml | 3 +- yb-voyager/cmd/common_test.go | 2 +- yb-voyager/cmd/exportDataStatus_test.go | 2 +- yb-voyager/go.mod | 11 +- yb-voyager/go.sum | 15 +- yb-voyager/src/callhome/diagnostics_test.go | 2 +- yb-voyager/src/cp/yugabyted/yugabyted_test.go | 22 ++- yb-voyager/src/datafile/descriptor_test.go | 2 +- yb-voyager/src/dbzm/status_test.go | 2 +- yb-voyager/src/metadb/metadataDB_test.go | 2 +- .../src/migassessment/assessmentDB_test.go | 2 +- yb-voyager/src/namereg/namereg_test.go | 2 +- yb-voyager/src/srcdb/main_test.go | 176 ++++++++++++++++++ yb-voyager/src/srcdb/mysql_test.go | 89 +++++++++ yb-voyager/src/srcdb/oracle_test.go | 80 ++++++++ yb-voyager/src/srcdb/postgres.go | 4 +- yb-voyager/src/srcdb/postgres_test.go | 136 ++++++++++++++ yb-voyager/src/srcdb/yugbaytedb_test.go | 136 ++++++++++++++ yb-voyager/src/tgtdb/conn_pool_test.go | 46 +---- yb-voyager/src/tgtdb/main_test.go | 139 ++++++++++++++ yb-voyager/src/tgtdb/postgres_test.go | 91 +++++++-- yb-voyager/src/tgtdb/yugabytedb_test.go | 93 +++++++-- yb-voyager/test/containers/helpers.go | 60 ++++++ yb-voyager/test/containers/mysql_container.go | 149 +++++++++++++++ .../test/containers/oracle_container.go | 107 +++++++++++ .../test/containers/postgres_container.go | 151 +++++++++++++++ .../containers/test_schemas/mysql_schema.sql | 6 + .../containers/test_schemas/oracle_schema.sql | 47 +++++ .../test_schemas/postgresql_schema.sql | 1 + .../test_schemas/yugabytedb_schema.sql | 1 + yb-voyager/test/containers/testcontainers.go | 143 ++++++++++++++ .../test/containers/yugabytedb_container.go | 129 +++++++++++++ .../utils/testutils.go} | 56 +++++- yb-voyager/testcontainers/testcontainers.go | 108 ----------- 36 files changed, 1862 insertions(+), 210 deletions(-) create mode 100644 .github/workflows/integration-tests.yml create mode 100644 yb-voyager/src/srcdb/main_test.go create mode 100644 yb-voyager/src/srcdb/mysql_test.go create mode 100644 yb-voyager/src/srcdb/oracle_test.go create mode 100644 yb-voyager/src/srcdb/postgres_test.go create mode 100644 yb-voyager/src/srcdb/yugbaytedb_test.go create mode 100644 yb-voyager/src/tgtdb/main_test.go create mode 100644 yb-voyager/test/containers/helpers.go create mode 100644 yb-voyager/test/containers/mysql_container.go create mode 100644 yb-voyager/test/containers/oracle_container.go create mode 100644 yb-voyager/test/containers/postgres_container.go create mode 100644 yb-voyager/test/containers/test_schemas/mysql_schema.sql create mode 100644 yb-voyager/test/containers/test_schemas/oracle_schema.sql create mode 100644 yb-voyager/test/containers/test_schemas/postgresql_schema.sql create mode 100644 yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql create mode 100644 yb-voyager/test/containers/testcontainers.go create mode 100644 yb-voyager/test/containers/yugabytedb_container.go rename yb-voyager/{src/testutils/testing_utils.go => test/utils/testutils.go} (89%) delete mode 100644 yb-voyager/testcontainers/testcontainers.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 96658551e3..ea81fd4e7c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,7 +8,7 @@ on: jobs: - build: + build-and-test: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' - name: Vet run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000000..b6b018be21 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,53 @@ +name: Go + +on: + push: + branches: ['main', '*.*-dev', '*.*.*-dev'] + pull_request: + branches: [main] + +env: + ORACLE_INSTANT_CLIENT_VERSION: "21.5.0.0.0-1" + +jobs: + integration-tests: + strategy: + fail-fast: false + + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.23.1" + + - name: Build + run: | + cd yb-voyager + go build -v ./... + + # required by godror driver used in the tests + - name: Install Oracle Instant Clients + run: | + # Download and install the YB APT repository package + wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/reporpms/yb-apt-repo_1.0.0_all.deb + sudo apt-get install -y ./yb-apt-repo_1.0.0_all.deb + sudo apt-get update -y + + # Install Oracle Instant Client packages using the defined version + sudo apt-get install -y oracle-instantclient-tools=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-basic=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-devel=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-jdbc=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-sqlplus=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + + # Clean up the YB APT repository package + sudo apt-get remove -y yb-apt-repo + rm -f yb-apt-repo_1.0.0_all.deb + + - name: Run Integration Tests + run: | + cd yb-voyager + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags 'integration' diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 2c965c5427..7db2e6260c 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -47,5 +47,4 @@ jobs: - name: Test Issues Against YB Version run: | cd yb-voyager - go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... - + go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index d0046ea7ad..6a5c462365 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -8,9 +8,9 @@ import ( "testing" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestAssessmentReportStructs(t *testing.T) { diff --git a/yb-voyager/cmd/exportDataStatus_test.go b/yb-voyager/cmd/exportDataStatus_test.go index beec85005c..692710aace 100644 --- a/yb-voyager/cmd/exportDataStatus_test.go +++ b/yb-voyager/cmd/exportDataStatus_test.go @@ -6,8 +6,8 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestExportSnapshotStatusStructs(t *testing.T) { diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index a8a306dbcb..8cf7a703f2 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -22,8 +22,8 @@ require ( github.com/gosuri/uilive v0.0.4 github.com/gosuri/uitable v0.0.4 github.com/hashicorp/go-version v1.7.0 - github.com/jackc/pgconn v1.13.0 - github.com/jackc/pgx/v4 v4.17.2 + github.com/jackc/pgconn v1.14.3 + github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.0.3 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 @@ -45,6 +45,7 @@ require ( golang.org/x/term v0.24.0 google.golang.org/api v0.169.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gotest.tools v2.2.0+incompatible ) require ( @@ -143,9 +144,9 @@ require ( github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pglogrepl v0.0.0-20231111135425-1627ab1b5780 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.1 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/magiconair/properties v1.8.7 github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 3b156d1c04..e68839fdc2 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1361,8 +1361,9 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pglogrepl v0.0.0-20231111135425-1627ab1b5780 h1:pNK2AKKIRC1MMMvpa6UiNtdtOebpiIloX7q2JZDkfsk= @@ -1380,22 +1381,26 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/pgx/v5 v5.0.3 h1:4flM5ecR/555F0EcnjdaZa6MhBU+nr0QbZIo5vaKjuM= github.com/jackc/pgx/v5 v5.0.3/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index a212004a05..9d63b562b5 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/google/uuid" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCallhomeStructs(t *testing.T) { diff --git a/yb-voyager/src/cp/yugabyted/yugabyted_test.go b/yb-voyager/src/cp/yugabyted/yugabyted_test.go index 1c6690999d..90bf0c5e25 100644 --- a/yb-voyager/src/cp/yugabyted/yugabyted_test.go +++ b/yb-voyager/src/cp/yugabyted/yugabyted_test.go @@ -14,29 +14,33 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v4/pgxpool" - _ "github.com/lib/pq" // PostgreSQL driver + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" controlPlane "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestYugabyteDTableSchema(t *testing.T) { ctx := context.Background() - // Start a YugabyteDB container - ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) + yugabyteDBContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err := yugabyteDBContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + defer testcontainers.TerminateAllContainers() assert.NoError(t, err, "Failed to start YugabyteDB container") - defer ybContainer.Terminate(ctx) // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + dsn := yugabyteDBContainer.GetConnectionString() + db, err := sql.Open("pgx", dsn) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Export the database connection string to env variable YUGABYTED_DB_CONN_STRING err = os.Setenv("YUGABYTED_DB_CONN_STRING", dsn) diff --git a/yb-voyager/src/datafile/descriptor_test.go b/yb-voyager/src/datafile/descriptor_test.go index dc617d0909..6092aba4c3 100644 --- a/yb-voyager/src/datafile/descriptor_test.go +++ b/yb-voyager/src/datafile/descriptor_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestDescriptorStructs(t *testing.T) { diff --git a/yb-voyager/src/dbzm/status_test.go b/yb-voyager/src/dbzm/status_test.go index 9b384e9ad4..753188dfd8 100644 --- a/yb-voyager/src/dbzm/status_test.go +++ b/yb-voyager/src/dbzm/status_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestExportStatusStructs(t *testing.T) { diff --git a/yb-voyager/src/metadb/metadataDB_test.go b/yb-voyager/src/metadb/metadataDB_test.go index 89648a26b7..b6c0126927 100644 --- a/yb-voyager/src/metadb/metadataDB_test.go +++ b/yb-voyager/src/metadb/metadataDB_test.go @@ -7,7 +7,7 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) // Test the initMetaDB function diff --git a/yb-voyager/src/migassessment/assessmentDB_test.go b/yb-voyager/src/migassessment/assessmentDB_test.go index 843d1cca06..854b52c8fd 100644 --- a/yb-voyager/src/migassessment/assessmentDB_test.go +++ b/yb-voyager/src/migassessment/assessmentDB_test.go @@ -7,7 +7,7 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestInitAssessmentDB(t *testing.T) { diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index 9d6b16c523..c785dbff02 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/require" "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) var oracleToYBNameRegistry = &NameRegistry{ diff --git a/yb-voyager/src/srcdb/main_test.go b/yb-voyager/src/srcdb/main_test.go new file mode 100644 index 0000000000..770855ad60 --- /dev/null +++ b/yb-voyager/src/srcdb/main_test.go @@ -0,0 +1,176 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "context" + "os" + "testing" + + _ "github.com/godror/godror" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" +) + +type TestDB struct { + testcontainers.TestContainer + *Source +} + +var ( + testPostgresSource *TestDB + testOracleSource *TestDB + testMySQLSource *TestDB + testYugabyteDBSource *TestDB +) + +func TestMain(m *testing.M) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer := testcontainers.NewTestContainer("postgresql", nil) + err := postgresContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start postgres container: %v", err) + } + host, port, err := postgresContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testPostgresSource = &TestDB{ + TestContainer: postgresContainer, + Source: &Source{ + DBType: "postgresql", + DBVersion: postgresContainer.GetConfig().DBVersion, + User: postgresContainer.GetConfig().User, + Password: postgresContainer.GetConfig().Password, + Schema: postgresContainer.GetConfig().Schema, + DBName: postgresContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + err = testPostgresSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to postgres database: %w", err) + } + defer testPostgresSource.DB().Disconnect() + + oracleContainer := testcontainers.NewTestContainer("oracle", nil) + err = oracleContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start oracle container: %v", err) + } + host, port, err = oracleContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + + testOracleSource = &TestDB{ + TestContainer: oracleContainer, + Source: &Source{ + DBType: "oracle", + DBVersion: oracleContainer.GetConfig().DBVersion, + User: oracleContainer.GetConfig().User, + Password: oracleContainer.GetConfig().Password, + Schema: oracleContainer.GetConfig().Schema, + DBName: oracleContainer.GetConfig().DBName, + Host: host, + Port: port, + }, + } + + err = testOracleSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to oracle database: %w", err) + } + defer testOracleSource.DB().Disconnect() + + mysqlContainer := testcontainers.NewTestContainer("mysql", nil) + err = mysqlContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start mysql container: %v", err) + } + host, port, err = mysqlContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testMySQLSource = &TestDB{ + TestContainer: mysqlContainer, + Source: &Source{ + DBType: "mysql", + DBVersion: mysqlContainer.GetConfig().DBVersion, + User: mysqlContainer.GetConfig().User, + Password: mysqlContainer.GetConfig().Password, + Schema: mysqlContainer.GetConfig().Schema, + DBName: mysqlContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + + err = testMySQLSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to mysql database: %w", err) + } + defer testMySQLSource.DB().Disconnect() + + yugabytedbContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err = yugabytedbContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + host, port, err = yugabytedbContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testYugabyteDBSource = &TestDB{ + TestContainer: yugabytedbContainer, + Source: &Source{ + DBType: "yugabytedb", + DBVersion: yugabytedbContainer.GetConfig().DBVersion, + User: yugabytedbContainer.GetConfig().User, + Password: yugabytedbContainer.GetConfig().Password, + Schema: yugabytedbContainer.GetConfig().Schema, + DBName: yugabytedbContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + + err = testYugabyteDBSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to yugabytedb database: %w", err) + } + defer testYugabyteDBSource.DB().Disconnect() + + // to avoid info level logs flooding the test output + log.SetLevel(log.WarnLevel) + + exitCode := m.Run() + + // cleanig up all the running containers + testcontainers.TerminateAllContainers() + + os.Exit(exitCode) +} diff --git a/yb-voyager/src/srcdb/mysql_test.go b/yb-voyager/src/srcdb/mysql_test.go new file mode 100644 index 0000000000..db74724162 --- /dev/null +++ b/yb-voyager/src/srcdb/mysql_test.go @@ -0,0 +1,89 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestMysqlGetAllTableNames(t *testing.T) { + testMySQLSource.ExecuteSqls( + `CREATE DATABASE test;`, + `CREATE TABLE test.foo ( + id INT PRIMARY KEY, + name VARCHAR(255) + );`, + `CREATE TABLE test.bar ( + id INT PRIMARY KEY, + name VARCHAR(255) + );`, + `CREATE TABLE test.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testMySQLSource.ExecuteSqls(`DROP DATABASE test;`) + + sqlname.SourceDBType = "mysql" + testMySQLSource.Source.DBName = "test" // used in query of GetAllTableNames() + + // Test GetAllTableNames + actualTables := testMySQLSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test", "foo"), + sqlname.NewSourceName("test", "bar"), + sqlname.NewSourceName("test", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +// TODO: Seems like a Bug somwhere, because now mysql.GetAllNonPkTables() as it is returning all the tables created in this test +// func TestMySQLGetNonPKTables(t *testing.T) { +// testMySQLSource.ExecuteSqls( +// `CREATE DATABASE test;`, +// `CREATE TABLE test.table1 ( +// id INT AUTO_INCREMENT PRIMARY KEY, +// name VARCHAR(100) +// );`, +// `CREATE TABLE test.table2 ( +// id INT AUTO_INCREMENT PRIMARY KEY, +// email VARCHAR(100) +// );`, +// `CREATE TABLE test.non_pk1( +// id INT, +// name VARCHAR(255) +// );`, +// `CREATE TABLE test.non_pk2( +// id INT, +// name VARCHAR(255) +// );`) +// defer testMySQLSource.ExecuteSqls(`DROP DATABASE test;`) + +// testMySQLSource.Source.DBName = "test" +// actualTables, err := testMySQLSource.DB().GetNonPKTables() +// assert.NilError(t, err, "Expected nil but non nil error: %v", err) + +// expectedTables := []string{"test.non_pk1", "test.non_pk2"} + +// testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +// } diff --git a/yb-voyager/src/srcdb/oracle_test.go b/yb-voyager/src/srcdb/oracle_test.go new file mode 100644 index 0000000000..9369b82575 --- /dev/null +++ b/yb-voyager/src/srcdb/oracle_test.go @@ -0,0 +1,80 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestOracleGetAllTableNames(t *testing.T) { + sqlname.SourceDBType = "oracle" + + // Test GetAllTableNames + actualTables := testOracleSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("YBVOYAGER", "foo"), + sqlname.NewSourceName("YBVOYAGER", "bar"), + sqlname.NewSourceName("YBVOYAGER", "table1"), + sqlname.NewSourceName("YBVOYAGER", "table2"), + sqlname.NewSourceName("YBVOYAGER", "unique_table"), + sqlname.NewSourceName("YBVOYAGER", "non_pk1"), + sqlname.NewSourceName("YBVOYAGER", "non_pk2"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestOracleGetTableToUniqueKeyColumnsMap(t *testing.T) { + objectName := sqlname.NewObjectName("oracle", "YBVOYAGER", "YBVOYAGER", "UNIQUE_TABLE") + + // Test GetTableToUniqueKeyColumnsMap + tableList := []sqlname.NameTuple{ + {CurrentName: objectName}, + } + uniqueKeys, err := testOracleSource.DB().GetTableToUniqueKeyColumnsMap(tableList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedKeys := map[string][]string{ + "UNIQUE_TABLE": {"EMAIL", "PHONE", "ADDRESS"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedKeys { + actualColumns, exists := uniqueKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestOracleGetNonPKTables(t *testing.T) { + actualTables, err := testOracleSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`YBVOYAGER."NON_PK1"`, `YBVOYAGER."NON_PK2"`} + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 4c90ffb5e6..ba2e21b720 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -940,6 +940,7 @@ var PG_QUERY_TO_CHECK_IF_TABLE_HAS_PK = `SELECT nspname AS schema_name, relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_constraint con ON con.conrelid = c.oid AND con.contype = 'p' +WHERE c.relkind = 'r' OR c.relkind = 'p' -- Only consider table objects GROUP BY schema_name, table_name HAVING nspname IN (%s);` func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { @@ -964,8 +965,9 @@ func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { if err != nil { return nil, fmt.Errorf("error in scanning query rows for primary key: %v", err) } - table := sqlname.NewSourceName(schemaName, fmt.Sprintf(`"%s"`, tableName)) + if pkCount == 0 { + table := sqlname.NewSourceName(schemaName, fmt.Sprintf(`"%s"`, tableName)) nonPKTables = append(nonPKTables, table.Qualified.Quoted) } } diff --git a/yb-voyager/src/srcdb/postgres_test.go b/yb-voyager/src/srcdb/postgres_test.go new file mode 100644 index 0000000000..41ac55d5ac --- /dev/null +++ b/yb-voyager/src/srcdb/postgres_test.go @@ -0,0 +1,136 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestPostgresGetAllTableNames(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + sqlname.SourceDBType = "postgresql" + testPostgresSource.Source.Schema = "test_schema" // used in query of GetAllTableNames() + + // Test GetAllTableNames + actualTables := testPostgresSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test_schema", "foo"), + sqlname.NewSourceName("test_schema", "bar"), + sqlname.NewSourceName("test_schema", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestPostgresGetTableToUniqueKeyColumnsMap(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + phone VARCHAR(20) UNIQUE, + address VARCHAR(255) UNIQUE + );`, + `INSERT INTO test_schema.unique_table (email, phone, address) VALUES + ('john@example.com', '1234567890', '123 Elm Street'), + ('jane@example.com', '0987654321', '456 Oak Avenue');`, + `CREATE TABLE test_schema.another_unique_table ( + user_id SERIAL PRIMARY KEY, + username VARCHAR(50) UNIQUE, + age INT + );`, + `CREATE UNIQUE INDEX idx_age ON test_schema.another_unique_table(age);`, + `INSERT INTO test_schema.another_unique_table (username, age) VALUES + ('user1', 30), + ('user2', 40);`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + uniqueTablesList := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "another_unique_table")}, + } + + actualUniqKeys, err := testPostgresSource.DB().GetTableToUniqueKeyColumnsMap(uniqueTablesList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedUniqKeys := map[string][]string{ + "test_schema.unique_table": {"email", "phone", "address"}, + "test_schema.another_unique_table": {"username", "age"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedUniqKeys { + actualColumns, exists := actualUniqKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestPostgresGetNonPKTables(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + actualTables, err := testPostgresSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`test_schema."non_pk2"`, `test_schema."non_pk1"`} // func returns table.Qualified.Quoted + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/srcdb/yugbaytedb_test.go b/yb-voyager/src/srcdb/yugbaytedb_test.go new file mode 100644 index 0000000000..4a7cf8e6f9 --- /dev/null +++ b/yb-voyager/src/srcdb/yugbaytedb_test.go @@ -0,0 +1,136 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestYugabyteGetAllTableNames(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + sqlname.SourceDBType = "postgresql" + testYugabyteDBSource.Source.Schema = "test_schema" + + // Test GetAllTableNames + actualTables := testYugabyteDBSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test_schema", "foo"), + sqlname.NewSourceName("test_schema", "bar"), + sqlname.NewSourceName("test_schema", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestYugabyteGetTableToUniqueKeyColumnsMap(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + phone VARCHAR(20) UNIQUE, + address VARCHAR(255) UNIQUE + );`, + `INSERT INTO test_schema.unique_table (email, phone, address) VALUES + ('john@example.com', '1234567890', '123 Elm Street'), + ('jane@example.com', '0987654321', '456 Oak Avenue');`, + `CREATE TABLE test_schema.another_unique_table ( + user_id SERIAL PRIMARY KEY, + username VARCHAR(50) UNIQUE, + age INT + );`, + `CREATE UNIQUE INDEX idx_age ON test_schema.another_unique_table(age);`, + `INSERT INTO test_schema.another_unique_table (username, age) VALUES + ('user1', 30), + ('user2', 40);`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + uniqueTablesList := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "another_unique_table")}, + } + + actualUniqKeys, err := testYugabyteDBSource.DB().GetTableToUniqueKeyColumnsMap(uniqueTablesList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedUniqKeys := map[string][]string{ + "test_schema.unique_table": {"email", "phone", "address"}, + "test_schema.another_unique_table": {"username", "age"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedUniqKeys { + actualColumns, exists := actualUniqKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestYugabyteGetNonPKTables(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + actualTables, err := testYugabyteDBSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`test_schema."non_pk2"`, `test_schema."non_pk1"`} // func returns table.Qualified.Quoted + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/tgtdb/conn_pool_test.go b/yb-voyager/src/tgtdb/conn_pool_test.go index 59fd4dad44..7d4ce43805 100644 --- a/yb-voyager/src/tgtdb/conn_pool_test.go +++ b/yb-voyager/src/tgtdb/conn_pool_test.go @@ -23,44 +23,18 @@ import ( "time" "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - - embeddedpostgres "github.com/fergusstrange/embedded-postgres" ) -// var postgres *embeddedpostgres.EmbeddedPostgres - -func setupPostgres(t *testing.T) *embeddedpostgres.EmbeddedPostgres { - postgres := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig(). - Username("postgres"). - Password("postgres"). - Database("test"). - Port(9876). - StartTimeout(30 * time.Second)) - err := postgres.Start() - if err != nil { - t.Fatal(err) - } - return postgres -} - -func shutdownPostgres(postgres *embeddedpostgres.EmbeddedPostgres, t *testing.T) { - err := postgres.Stop() - if err != nil { - t.Fatal(err) - } -} - func TestBasic(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10. size := 10 connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: size, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -88,8 +62,6 @@ func dummyProcess(pool *ConnectionPool, milliseconds int, wg *sync.WaitGroup) { } func TestIncreaseConnectionsUptoMax(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -97,7 +69,7 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -105,7 +77,7 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { // WHEN: multiple goroutines acquire connection, perform some operation // and release connection back to pool - // WHEN: we keep increasing the connnections upto the max.. + // WHEN: we keep increasing the connections upto the max.. var wg sync.WaitGroup wg.Add(1) @@ -133,8 +105,6 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { } func TestDecreaseConnectionsUptoMin(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -142,7 +112,7 @@ func TestDecreaseConnectionsUptoMin(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -178,8 +148,6 @@ func TestDecreaseConnectionsUptoMin(t *testing.T) { } func TestUpdateConnectionsRandom(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -187,7 +155,7 @@ func TestUpdateConnectionsRandom(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -207,7 +175,7 @@ func TestUpdateConnectionsRandom(t *testing.T) { if pool.size+randomNumber < 1 || (pool.size+randomNumber > pool.params.NumMaxConnections) { continue } - fmt.Printf("i=%d, updating by %d. New pool size expected = %d\n", i, randomNumber, *expectedFinalSize+randomNumber) + log.Infof("i=%d, updating by %d. New pool size expected = %d\n", i, randomNumber, *expectedFinalSize+randomNumber) err := pool.UpdateNumConnections(randomNumber) assert.NoError(t, err) time.Sleep(10 * time.Millisecond) diff --git a/yb-voyager/src/tgtdb/main_test.go b/yb-voyager/src/tgtdb/main_test.go new file mode 100644 index 0000000000..46cc2150e2 --- /dev/null +++ b/yb-voyager/src/tgtdb/main_test.go @@ -0,0 +1,139 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tgtdb + +import ( + "context" + "os" + "testing" + + _ "github.com/godror/godror" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" +) + +type TestDB struct { + testcontainers.TestContainer + TargetDB +} + +var ( + testPostgresTarget *TestDB + testOracleTarget *TestDB + testYugabyteDBTarget *TestDB +) + +func TestMain(m *testing.M) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer := testcontainers.NewTestContainer("postgresql", nil) + err := postgresContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start postgres container: %v", err) + } + host, port, err := postgresContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testPostgresTarget = &TestDB{ + TestContainer: postgresContainer, + TargetDB: NewTargetDB(&TargetConf{ + TargetDBType: "postgresql", + DBVersion: postgresContainer.GetConfig().DBVersion, + User: postgresContainer.GetConfig().User, + Password: postgresContainer.GetConfig().Password, + Schema: postgresContainer.GetConfig().Schema, + DBName: postgresContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }), + } + + err = testPostgresTarget.Init() + if err != nil { + utils.ErrExit("Failed to connect to postgres database: %w", err) + } + defer testPostgresTarget.Finalize() + + // oracleContainer := testcontainers.NewTestContainer("oracle", nil) + // _ = oracleContainer.Start(ctx) + // host, port, err = oracleContainer.GetHostPort() + // if err != nil { + // utils.ErrExit("%v", err) + // } + // testOracleTarget = &TestDB2{ + // Container: oracleContainer, + // TargetDB: NewTargetDB(&TargetConf{ + // TargetDBType: "oracle", + // DBVersion: oracleContainer.GetConfig().DBVersion, + // User: oracleContainer.GetConfig().User, + // Password: oracleContainer.GetConfig().Password, + // Schema: oracleContainer.GetConfig().Schema, + // DBName: oracleContainer.GetConfig().DBName, + // Host: host, + // Port: port, + // }), + // } + + // err = testOracleTarget.Init() + // if err != nil { + // utils.ErrExit("Failed to connect to oracle database: %w", err) + // } + // defer testOracleTarget.Finalize() + + yugabytedbContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err = yugabytedbContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + host, port, err = yugabytedbContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testYugabyteDBTarget = &TestDB{ + TestContainer: yugabytedbContainer, + TargetDB: NewTargetDB(&TargetConf{ + TargetDBType: "yugabytedb", + DBVersion: yugabytedbContainer.GetConfig().DBVersion, + User: yugabytedbContainer.GetConfig().User, + Password: yugabytedbContainer.GetConfig().Password, + Schema: yugabytedbContainer.GetConfig().Schema, + DBName: yugabytedbContainer.GetConfig().DBName, + Host: host, + Port: port, + }), + } + + err = testYugabyteDBTarget.Init() + if err != nil { + utils.ErrExit("Failed to connect to yugabytedb database: %w", err) + } + defer testYugabyteDBTarget.Finalize() + + // to avoid info level logs flooding the test output + log.SetLevel(log.WarnLevel) + + exitCode := m.Run() + + // cleaning up all the running containers + testcontainers.TerminateAllContainers() + + os.Exit(exitCode) +} diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go index 251ced5a63..e0e9bd60c5 100644 --- a/yb-voyager/src/tgtdb/postgres_test.go +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -1,33 +1,39 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( - "context" "database/sql" "fmt" "strings" "testing" + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCreateVoyagerSchemaPG(t *testing.T) { - ctx := context.Background() - - // Start a PostgreSQL container - pgContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.POSTGRESQL) - assert.NoError(t, err, "Failed to start PostgreSQL container") - defer pgContainer.Terminate(ctx) - - // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=testuser password=testpassword dbname=testdb sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("pgx", testPostgresTarget.GetConnectionString()) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Initialize the TargetYugabyteDB instance @@ -80,3 +86,60 @@ func TestCreateVoyagerSchemaPG(t *testing.T) { }) } } + +func TestPostgresGetNonEmptyTables(t *testing.T) { + testPostgresTarget.ExecuteSqls( + `CREATE SCHEMA test_schema`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(100), + phone VARCHAR(100), + address VARCHAR(255), + UNIQUE (email, phone) -- Unique constraint on combination of columns + );`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testPostgresTarget.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + tables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "bar")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "table1")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "table2")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "non_pk1")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "non_pk2")}, + } + + expectedTables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "bar")}, + } + + actualTables := testPostgresTarget.GetNonEmptyTables(tables) + testutils.AssertEqualNameTuplesSlice(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go index af22664460..755ed46d67 100644 --- a/yb-voyager/src/tgtdb/yugabytedb_test.go +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -1,33 +1,40 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( - "context" "database/sql" "fmt" "strings" "testing" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCreateVoyagerSchemaYB(t *testing.T) { - ctx := context.Background() - - // Start a YugabyteDB container - ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) - assert.NoError(t, err, "Failed to start YugabyteDB container") - defer ybContainer.Terminate(ctx) - - // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("pgx", testYugabyteDBTarget.GetConnectionString()) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Initialize the TargetYugabyteDB instance @@ -80,3 +87,61 @@ func TestCreateVoyagerSchemaYB(t *testing.T) { }) } } + +func TestYugabyteGetNonEmptyTables(t *testing.T) { + testYugabyteDBTarget.ExecuteSqls( + `CREATE SCHEMA test_schema`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(100), + phone VARCHAR(100), + address VARCHAR(255), + UNIQUE (email, phone) + );`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBTarget.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + tables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "bar")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "table1")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "table2")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "non_pk1")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "non_pk2")}, + } + + expectedTables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "bar")}, + } + + actualTables := testYugabyteDBTarget.GetNonEmptyTables(tables) + log.Infof("non empty tables: %+v\n", actualTables) + testutils.AssertEqualNameTuplesSlice(t, expectedTables, actualTables) +} diff --git a/yb-voyager/test/containers/helpers.go b/yb-voyager/test/containers/helpers.go new file mode 100644 index 0000000000..cd3afee88d --- /dev/null +++ b/yb-voyager/test/containers/helpers.go @@ -0,0 +1,60 @@ +package testcontainers + +import ( + "context" + _ "embed" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + + "github.com/testcontainers/testcontainers-go" +) + +const ( + DEFAULT_PG_PORT = "5432" + DEFAULT_YB_PORT = "5433" + DEFAULT_ORACLE_PORT = "1521" + DEFAULT_MYSQL_PORT = "3306" + + POSTGRESQL = "postgresql" + YUGABYTEDB = "yugabytedb" + ORACLE = "oracle" + MYSQL = "mysql" +) + +//go:embed test_schemas/postgresql_schema.sql +var postgresInitSchemaFile []byte + +//go:embed test_schemas/oracle_schema.sql +var oracleInitSchemaFile []byte + +//go:embed test_schemas/mysql_schema.sql +var mysqlInitSchemaFile []byte + +//go:embed test_schemas/yugabytedb_schema.sql +var yugabytedbInitSchemaFile []byte + +func printContainerLogs(container testcontainers.Container) { + if container == nil { + log.Printf("Cannot fetch logs: container is nil") + return + } + + containerID := container.GetContainerID() + logs, err := container.Logs(context.Background()) + if err != nil { + log.Printf("Error fetching logs for container %s: %v", containerID, err) + return + } + defer logs.Close() + + // Read the logs + logData, err := io.ReadAll(logs) + if err != nil { + log.Printf("Error reading logs for container %s: %v", containerID, err) + return + } + + fmt.Printf("=== Logs for container %s ===\n%s\n=== End of Logs for container %s ===\n", containerID, string(logData), containerID) +} diff --git a/yb-voyager/test/containers/mysql_container.go b/yb-voyager/test/containers/mysql_container.go new file mode 100644 index 0000000000..c2d8930cff --- /dev/null +++ b/yb-voyager/test/containers/mysql_container.go @@ -0,0 +1,149 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type MysqlContainer struct { + ContainerConfig + container testcontainers.Container + db *sql.DB +} + +func (ms *MysqlContainer) Start(ctx context.Context) (err error) { + if ms.container != nil { + utils.PrintAndLog("Mysql-%s container already running", ms.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "mysql_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(mysqlInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones + Image: fmt.Sprintf("mysql:%s", ms.DBVersion), + ExposedPorts: []string{"3306/tcp"}, + Env: map[string]string{ + "MYSQL_ROOT_PASSWORD": ms.Password, + "MYSQL_USER": ms.User, + "MYSQL_PASSWORD": ms.Password, + "MYSQL_DATABASE": ms.DBName, + }, + WaitingFor: wait.ForListeningPort("3306/tcp").WithStartupTimeout(2 * time.Minute).WithPollInterval(5 * time.Second), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/mysql_schema.sql", + FileMode: 0755, + }, + }, + } + + ms.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(ms.container) + if err != nil { + return err + } + + dsn := ms.GetConnectionString() + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to open mysql connection: %w", err) + } + + if err = db.Ping(); err != nil { + db.Close() + return fmt.Errorf("failed to ping mysql after connection: %w", err) + } + + // Store the DB connection for reuse + ms.db = db + + return nil +} + +func (ms *MysqlContainer) Terminate(ctx context.Context) { + if ms == nil { + return + } + + // Close the DB connection if it exists + if ms.db != nil { + if err := ms.db.Close(); err != nil { + log.Errorf("failed to close mysql db connection: %v", err) + } + } + + err := ms.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate mysql container: %v", err) + } +} + +func (ms *MysqlContainer) GetHostPort() (string, int, error) { + if ms.container == nil { + return "", -1, fmt.Errorf("mysql container is not started: nil") + } + + ctx := context.Background() + host, err := ms.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for mysql container: %w", err) + } + + port, err := ms.container.MappedPort(ctx, nat.Port(DEFAULT_MYSQL_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for mysql container: %w", err) + } + + return host, port.Int(), nil +} + +func (ms *MysqlContainer) GetConfig() ContainerConfig { + return ms.ContainerConfig +} + +func (ms *MysqlContainer) GetConnectionString() string { + host, port, err := ms.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for mysql connection string: %v", err) + } + + // DSN format: user:password@tcp(host:port)/dbname + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + ms.User, ms.Password, host, port, ms.DBName) +} + +func (ms *MysqlContainer) ExecuteSqls(sqls ...string) { + if ms.db == nil { + utils.ErrExit("db connection not initialized for mysql container") + } + + for _, sqlStmt := range sqls { + _, err := ms.db.Exec(sqlStmt) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sqlStmt, err) + } + } +} diff --git a/yb-voyager/test/containers/oracle_container.go b/yb-voyager/test/containers/oracle_container.go new file mode 100644 index 0000000000..8fb8218de2 --- /dev/null +++ b/yb-voyager/test/containers/oracle_container.go @@ -0,0 +1,107 @@ +package testcontainers + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type OracleContainer struct { + ContainerConfig + container testcontainers.Container +} + +func (ora *OracleContainer) Start(ctx context.Context) (err error) { + if ora.container != nil { + utils.PrintAndLog("Oracle-%s container already running", ora.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "oracle_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(oracleInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + // refer: https://hub.docker.com/r/gvenzl/oracle-xe + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones (No license issue) + Image: fmt.Sprintf("gvenzl/oracle-xe:%s", ora.DBVersion), + ExposedPorts: []string{"1521/tcp"}, + Env: map[string]string{ + "ORACLE_PASSWORD": ora.Password, // for SYS user + "ORACLE_DATABASE": ora.DBName, + "APP_USER": ora.User, + "APP_USER_PASSWORD": ora.Password, + }, + WaitingFor: wait.ForLog("DATABASE IS READY TO USE").WithStartupTimeout(2 * time.Minute).WithPollInterval(5 * time.Second), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/oracle_schema.sql", + FileMode: 0755, + }, + }, + } + + ora.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(ora.container) + return err +} + +func (ora *OracleContainer) Terminate(ctx context.Context) { + if ora == nil { + return + } + + err := ora.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate oracle container: %v", err) + } +} + +func (ora *OracleContainer) GetHostPort() (string, int, error) { + if ora.container == nil { + return "", -1, fmt.Errorf("oracle container is not started: nil") + } + + ctx := context.Background() + host, err := ora.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for oracle container: %w", err) + } + + port, err := ora.container.MappedPort(ctx, nat.Port(DEFAULT_ORACLE_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for oracle container: %w", err) + } + + return host, port.Int(), nil +} + +func (ora *OracleContainer) GetConfig() ContainerConfig { + return ora.ContainerConfig +} + +func (ora *OracleContainer) GetConnectionString() string { + panic("GetConnectionString() not implemented yet for oracle") +} + +func (ora *OracleContainer) ExecuteSqls(sqls ...string) { + +} diff --git a/yb-voyager/test/containers/postgres_container.go b/yb-voyager/test/containers/postgres_container.go new file mode 100644 index 0000000000..539763f7de --- /dev/null +++ b/yb-voyager/test/containers/postgres_container.go @@ -0,0 +1,151 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type PostgresContainer struct { + ContainerConfig + container testcontainers.Container + db *sql.DB +} + +func (pg *PostgresContainer) Start(ctx context.Context) (err error) { + if pg.container != nil { + utils.PrintAndLog("Postgres-%s container already running", pg.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "postgresql_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(postgresInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones + Image: fmt.Sprintf("postgres:%s", pg.DBVersion), + ExposedPorts: []string{"5432/tcp"}, + Env: map[string]string{ + "POSTGRES_USER": pg.User, + "POSTGRES_PASSWORD": pg.Password, + "POSTGRES_DB": pg.DBName, // NOTE: PG image makes the database with same name as user if not specific + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("5432/tcp").WithStartupTimeout(2*time.Minute).WithPollInterval(5*time.Second), + wait.ForLog("database system is ready to accept connections").WithStartupTimeout(3*time.Minute).WithPollInterval(5*time.Second), + ), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/postgresql_schema.sql", + FileMode: 0755, + }, + }, + } + + pg.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + + printContainerLogs(pg.container) + if err != nil { + return err + } + + dsn := pg.GetConnectionString() + db, err := sql.Open("pgx", dsn) + if err != nil { + return fmt.Errorf("failed to open postgres connection: %w", err) + } + + if err := db.Ping(); err != nil { + db.Close() + pg.container.Terminate(ctx) + return fmt.Errorf("failed to ping postgres after connection: %w", err) + } + + // Store the DB connection for reuse + pg.db = db + return nil +} + +func (pg *PostgresContainer) Terminate(ctx context.Context) { + if pg == nil { + return + } + + // Close the DB connection if it exists + if pg.db != nil { + if err := pg.db.Close(); err != nil { + log.Errorf("failed to close postgres db connection: %v", err) + } + } + + err := pg.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate postgres container: %v", err) + } +} + +func (pg *PostgresContainer) GetHostPort() (string, int, error) { + if pg.container == nil { + return "", -1, fmt.Errorf("postgres container is not started: nil") + } + + ctx := context.Background() + host, err := pg.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for postgres container: %w", err) + } + + port, err := pg.container.MappedPort(ctx, nat.Port(DEFAULT_PG_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for postgres container: %w", err) + } + + return host, port.Int(), nil +} + +func (pg *PostgresContainer) GetConfig() ContainerConfig { + return pg.ContainerConfig +} + +func (pg *PostgresContainer) GetConnectionString() string { + config := pg.GetConfig() + host, port, err := pg.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for postgres connection string: %v", err) + } + + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", config.User, config.Password, host, port, config.DBName) +} + +func (pg *PostgresContainer) ExecuteSqls(sqls ...string) { + if pg.db == nil { + utils.ErrExit("db connection not initialized for postgres container") + } + + for _, sqlStmt := range sqls { + _, err := pg.db.Exec(sqlStmt) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sqlStmt, err) + } + } +} diff --git a/yb-voyager/test/containers/test_schemas/mysql_schema.sql b/yb-voyager/test/containers/test_schemas/mysql_schema.sql new file mode 100644 index 0000000000..e72f4bcb17 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/mysql_schema.sql @@ -0,0 +1,6 @@ +-- TODO: create user as per User creation steps in docs and use that in tests + +-- Grant CREATE, ALTER, DROP privileges globally to 'ybvoyager' +GRANT CREATE, ALTER, DROP ON *.* TO 'ybvoyager'@'%' WITH GRANT OPTION; +-- Apply the changes +FLUSH PRIVILEGES; \ No newline at end of file diff --git a/yb-voyager/test/containers/test_schemas/oracle_schema.sql b/yb-voyager/test/containers/test_schemas/oracle_schema.sql new file mode 100644 index 0000000000..95b3a3a9b4 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/oracle_schema.sql @@ -0,0 +1,47 @@ +-- TODO: create user as per User creation steps in docs and use that in tests + +-- Used ORACLE_DATABASE=DMS i.e. pluggable database to create APP_USER +ALTER SESSION SET CONTAINER = "DMS"; + + +-- creating tables under YBVOYAGER schema, same as APP_USER +CREATE TABLE YBVOYAGER.foo ( + id NUMBER PRIMARY KEY, + name VARCHAR2(255) +); + +CREATE TABLE YBVOYAGER.bar ( + id NUMBER PRIMARY KEY, + name VARCHAR2(255) +); + +CREATE TABLE YBVOYAGER.unique_table ( + id NUMBER PRIMARY KEY, + email VARCHAR2(100), + phone VARCHAR2(100), + address VARCHAR2(255), + CONSTRAINT email_phone_unq UNIQUE (email, phone) +); + +CREATE UNIQUE INDEX YBVOYAGER.unique_address_idx ON YBVOYAGER.unique_table (address); + +CREATE TABLE YBVOYAGER.table1 ( + id NUMBER PRIMARY KEY, + name VARCHAR2(100) +); + +CREATE TABLE YBVOYAGER.table2 ( + id NUMBER PRIMARY KEY, + email VARCHAR2(100) +); + + +CREATE TABLE YBVOYAGER.non_pk1 ( + id NUMBER, + name VARCHAR2(10) +); + +CREATE TABLE YBVOYAGER.non_pk2 ( + id NUMBER, + name VARCHAR2(10) +); \ No newline at end of file diff --git a/yb-voyager/test/containers/test_schemas/postgresql_schema.sql b/yb-voyager/test/containers/test_schemas/postgresql_schema.sql new file mode 100644 index 0000000000..36bda657a5 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/postgresql_schema.sql @@ -0,0 +1 @@ +-- TODO: create source migration user as per User creation steps in docs and use that in tests diff --git a/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql b/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql new file mode 100644 index 0000000000..c36ddc5b93 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql @@ -0,0 +1 @@ +-- TODO: create user as per User creation steps in docs and use that in tests diff --git a/yb-voyager/test/containers/testcontainers.go b/yb-voyager/test/containers/testcontainers.go new file mode 100644 index 0000000000..c9fce4b15c --- /dev/null +++ b/yb-voyager/test/containers/testcontainers.go @@ -0,0 +1,143 @@ +package testcontainers + +import ( + "context" + "fmt" + "sync" + + "github.com/samber/lo" + log "github.com/sirupsen/logrus" +) + +// containerRegistry to ensure one container per database(dbtype+version) [Singleton Pattern] +// Limitation - go test spawns different process for running tests of each package, hence the containers won't be shared across packages. +var ( + containerRegistry = make(map[string]TestContainer) + registryMutex sync.Mutex +) + +type TestContainer interface { + Start(ctx context.Context) error + Terminate(ctx context.Context) + GetHostPort() (string, int, error) + GetConfig() ContainerConfig + GetConnectionString() string + /* + TODOs + // Function to run sql script for a specific test case + SetupSqlScript(scriptName string, dbName string) error + + // Add Capability to run multiple versions of a dbtype parallely + */ + ExecuteSqls(sqls ...string) +} + +type ContainerConfig struct { + DBVersion string + User string + Password string + DBName string + Schema string +} + +func NewTestContainer(dbType string, containerConfig *ContainerConfig) TestContainer { + registryMutex.Lock() + defer registryMutex.Unlock() + + // initialise containerConfig struct if nothing is provided + if containerConfig == nil { + containerConfig = &ContainerConfig{} + } + setContainerConfigDefaultsIfNotProvided(dbType, containerConfig) + + // check if container is already created after fetching default configs + containerName := fmt.Sprintf("%s-%s", dbType, containerConfig.DBVersion) + if container, exists := containerRegistry[containerName]; exists { + log.Infof("container '%s' already exists in the registry", containerName) + return container + } + + var testContainer TestContainer + switch dbType { + case POSTGRESQL: + testContainer = &PostgresContainer{ + ContainerConfig: *containerConfig, + } + case YUGABYTEDB: + testContainer = &YugabyteDBContainer{ + ContainerConfig: *containerConfig, + } + case ORACLE: + testContainer = &OracleContainer{ + ContainerConfig: *containerConfig, + } + case MYSQL: + testContainer = &MysqlContainer{ + ContainerConfig: *containerConfig, + } + default: + panic(fmt.Sprintf("unsupported db type '%q' for creating test container\n", dbType)) + } + + containerRegistry[containerName] = testContainer + return testContainer +} + +/* +Challenges in golang for running this a teardown step +1. In golang when you execute go test in the top level folder it executes all the tests one by one. +2. Where each defined package, can have its TestMain() which can control the setup and teardown steps for that package +3. There is no way to run these before/after the tests of first/last package in codebase + +Potential solution: Implement a counter(total=number_of_package) based logic to execute teardown(i.e. TerminateAllContainers() in our case) +Figure out the best solution. + +For now we can rely on TestContainer ryuk(the container repear), which terminates all the containers after the process exits. +But the test framework should have capability of terminating all containers at the end. +*/ +func TerminateAllContainers() { + registryMutex.Lock() + defer registryMutex.Unlock() + + ctx := context.Background() + for name, container := range containerRegistry { + log.Infof("terminating the container '%s'", name) + container.Terminate(ctx) + } +} + +func setContainerConfigDefaultsIfNotProvided(dbType string, config *ContainerConfig) { + // TODO: discuss and decide the default DBVersion values for each dbtype + + switch dbType { + case POSTGRESQL: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "11", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "public", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "postgres", config.DBName) + + case YUGABYTEDB: + config.User = lo.Ternary(config.User == "", "yugabyte", config.User) // ybdb docker doesn't create specified user + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "2.20.7.1-b10", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "public", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "yugabyte", config.DBName) + + case ORACLE: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "21", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "YBVOYAGER", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "DMS", config.DBName) + + case MYSQL: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "8.4", config.DBVersion) + config.DBName = lo.Ternary(config.DBName == "", "dms", config.DBName) + + default: + panic(fmt.Sprintf("unsupported db type '%q' for creating test container\n", dbType)) + } +} diff --git a/yb-voyager/test/containers/yugabytedb_container.go b/yb-voyager/test/containers/yugabytedb_container.go new file mode 100644 index 0000000000..822a7f6945 --- /dev/null +++ b/yb-voyager/test/containers/yugabytedb_container.go @@ -0,0 +1,129 @@ +package testcontainers + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + "github.com/jackc/pgx/v5" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type YugabyteDBContainer struct { + ContainerConfig + container testcontainers.Container +} + +func (yb *YugabyteDBContainer) Start(ctx context.Context) (err error) { + if yb.container != nil { + utils.PrintAndLog("YugabyteDB-%s container already running", yb.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "yugabytedb_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(yugabytedbInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + // this will create a 1 Node RF-1 cluster + req := testcontainers.ContainerRequest{ + Image: fmt.Sprintf("yugabytedb/yugabyte:%s", yb.DBVersion), + ExposedPorts: []string{"5433/tcp", "15433/tcp", "7000/tcp", "9000/tcp", "9042/tcp"}, + Cmd: []string{ + "bin/yugabyted", + "start", + "--daemon=false", + "--ui=false", + "--initial_scripts_dir=/home/yugabyte/initial-scripts", + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("5433/tcp").WithStartupTimeout(2*time.Minute).WithPollInterval(5*time.Second), + wait.ForLog("Data placement constraint successfully verified").WithStartupTimeout(3*time.Minute).WithPollInterval(1*time.Second), + ), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "/home/yugabyte/initial-scripts/yugabytedb_schema.sql", + FileMode: 0755, + }, + }, + } + + yb.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(yb.container) + return err +} + +func (yb *YugabyteDBContainer) Terminate(ctx context.Context) { + if yb == nil { + return + } + + err := yb.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate yugabytedb container: %v", err) + } +} + +func (yb *YugabyteDBContainer) GetHostPort() (string, int, error) { + if yb.container == nil { + return "", -1, fmt.Errorf("yugabytedb container is not started: nil") + } + + ctx := context.Background() + host, err := yb.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for yugabytedb container: %w", err) + } + + port, err := yb.container.MappedPort(ctx, nat.Port(DEFAULT_YB_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for yugabytedb container: %w", err) + } + + return host, port.Int(), nil +} + +func (yb *YugabyteDBContainer) GetConfig() ContainerConfig { + return yb.ContainerConfig +} + +func (yb *YugabyteDBContainer) GetConnectionString() string { + config := yb.GetConfig() + host, port, err := yb.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for yugabytedb connection string: %v", err) + } + + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", config.User, config.Password, host, port, config.DBName) +} + +func (yb *YugabyteDBContainer) ExecuteSqls(sqls ...string) { + connStr := yb.GetConnectionString() + conn, err := pgx.Connect(context.Background(), connStr) + if err != nil { + utils.ErrExit("failed to connect postgres for executing sqls: %w", err) + } + defer conn.Close(context.Background()) + + for _, sql := range sqls { + _, err := conn.Exec(context.Background(), sql) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sql, err) + } + } +} diff --git a/yb-voyager/src/testutils/testing_utils.go b/yb-voyager/test/utils/testutils.go similarity index 89% rename from yb-voyager/src/testutils/testing_utils.go rename to yb-voyager/test/utils/testutils.go index 2a10007dee..74c30bc446 100644 --- a/yb-voyager/src/testutils/testing_utils.go +++ b/yb-voyager/test/utils/testutils.go @@ -6,11 +6,14 @@ import ( "os" "reflect" "regexp" + "sort" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) type ColumnPropertiesSqlite struct { @@ -201,10 +204,10 @@ func CheckTableStructurePG(t *testing.T, db *sql.DB, schema, table string, expec func checkPrimaryKeyOfTablePG(t *testing.T, db *sql.DB, schema, table string, expectedColumns map[string]ColumnPropertiesPG) { // Validate primary keys queryPrimaryKeys := ` - SELECT conrelid::regclass AS table_name, - conname AS primary_key, + SELECT conrelid::regclass AS table_name, + conname AS primary_key, pg_get_constraintdef(oid) - FROM pg_constraint + FROM pg_constraint WHERE contype = 'p' -- 'p' indicates primary key AND conrelid::regclass::text = $1 ORDER BY conrelid::regclass::text, contype DESC;` @@ -350,3 +353,50 @@ func CheckTableExistencePG(t *testing.T, db *sql.DB, schema string, expectedTabl } } } + +// === assertion helper functions +func AssertEqualStringSlices(t *testing.T, expected, actual []string) { + t.Helper() + if len(expected) != len(actual) { + t.Errorf("Mismatch in slice length. Expected: %v, Actual: %v", expected, actual) + } + + sort.Strings(expected) + sort.Strings(actual) + assert.Equal(t, expected, actual) +} + +func AssertEqualSourceNameSlices(t *testing.T, expected, actual []*sqlname.SourceName) { + SortSourceNames(expected) + SortSourceNames(actual) + assert.Equal(t, expected, actual) +} + +func SortSourceNames(tables []*sqlname.SourceName) { + sort.Slice(tables, func(i, j int) bool { + return tables[i].Qualified.MinQuoted < tables[j].Qualified.MinQuoted + }) +} + +func AssertEqualNameTuplesSlice(t *testing.T, expected, actual []sqlname.NameTuple) { + sortNameTuples(expected) + sortNameTuples(actual) + assert.Equal(t, expected, actual) +} + +func sortNameTuples(tables []sqlname.NameTuple) { + sort.Slice(tables, func(i, j int) bool { + return tables[i].ForOutput() < tables[j].ForOutput() + }) +} + +// waitForDBConnection waits until the database is ready for connections. +func WaitForDBToBeReady(db *sql.DB) error { + for i := 0; i < 12; i++ { + if err := db.Ping(); err == nil { + return nil + } + time.Sleep(5 * time.Second) + } + return fmt.Errorf("database did not become ready in time") +} diff --git a/yb-voyager/testcontainers/testcontainers.go b/yb-voyager/testcontainers/testcontainers.go deleted file mode 100644 index 336d7b2e30..0000000000 --- a/yb-voyager/testcontainers/testcontainers.go +++ /dev/null @@ -1,108 +0,0 @@ -package testcontainers - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const ( - POSTGRESQL = "postgresql" - YUGABYTEDB = "yugabytedb" -) - -func StartDBContainer(ctx context.Context, dbType string) (testcontainers.Container, string, nat.Port, error) { - switch dbType { - case POSTGRESQL: - return startPostgresContainer(ctx) - case YUGABYTEDB: - return startYugabyteDBContainer(ctx) - default: - return nil, "", "", fmt.Errorf("unsupported database type: %s", dbType) - } -} - -func startPostgresContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { - // Create a PostgreSQL TestContainer - req := testcontainers.ContainerRequest{ - Image: "postgres:latest", // Use the latest PostgreSQL image - ExposedPorts: []string{"5432/tcp"}, - Env: map[string]string{ - "POSTGRES_USER": "testuser", // Set PostgreSQL username - "POSTGRES_PASSWORD": "testpassword", // Set PostgreSQL password - "POSTGRES_DB": "testdb", // Set PostgreSQL database name - }, - WaitingFor: wait.ForListeningPort("5432/tcp").WithStartupTimeout(30 * 1e9), // Wait for PostgreSQL to be ready - } - - // Start the container - pgContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - - if err != nil { - return nil, "", "", err - } - - // Get the container's host and port - host, err = pgContainer.Host(ctx) - if err != nil { - return nil, "", "", err - } - - port, err = pgContainer.MappedPort(ctx, "5432") - if err != nil { - return nil, "", "", err - } - - return pgContainer, host, port, nil -} - -func startYugabyteDBContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { - // Create a YugabyteDB TestContainer - req := testcontainers.ContainerRequest{ - Image: "yugabytedb/yugabyte:latest", - ExposedPorts: []string{"5433/tcp"}, - WaitingFor: wait.ForListeningPort("5433/tcp"), - Cmd: []string{"bin/yugabyted", "start", "--daemon=false"}, - } - - // Start the container - ybContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - return nil, "", "", err - } - - // Get the container's host and port - host, err = ybContainer.Host(ctx) - if err != nil { - return nil, "", "", err - } - - port, err = ybContainer.MappedPort(ctx, "5433") - if err != nil { - return nil, "", "", err - } - - return ybContainer, host, port, nil -} - -// waitForDBConnection waits until the database is ready for connections. -func WaitForDBToBeReady(db *sql.DB) error { - for i := 0; i < 12; i++ { - if err := db.Ping(); err == nil { - return nil - } - time.Sleep(5 * time.Second) - } - return fmt.Errorf("database did not become ready in time") -} From f19b943d9e08929dce609fab1ecd8b39cb90fc57 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:08:42 +0530 Subject: [PATCH 078/105] Added a new detector for catching unsupported COPY command structures in PG (#2106) --- .../expectedAssessmentReport.json | 35 +++++++-- .../unsupported_query_constructs.sql | 30 +++++++- yb-voyager/src/query/queryissue/constants.go | 2 + yb-voyager/src/query/queryissue/detectors.go | 77 +++++++++++++++---- yb-voyager/src/query/queryissue/issues_dml.go | 26 +++++++ .../src/query/queryissue/issues_dml_test.go | 28 +++++++ .../query/queryissue/parser_issue_detector.go | 3 +- .../queryissue/parser_issue_detector_test.go | 39 ++++++++++ .../src/query/queryparser/helpers_protomsg.go | 10 ++- .../src/query/queryparser/traversal_proto.go | 1 + 10 files changed, 227 insertions(+), 24 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 1057ad9fd8..5a861d6410 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 28, + "TotalCount": 29, "InvalidCount": 0, - "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees_employee_id_seq, public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 67, + "TotalCount": 68, "InvalidCount": 23, - "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" + "ObjectNames": "public.employees3, public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" }, { "ObjectType": "INDEX", @@ -170,9 +170,10 @@ "test_views.view_table1", "public.library_nested", "public.orders_lateral", - "public.employees" + "public.employees", + "public.employees3" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 73 objects (65 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 74 objects (66 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -1983,6 +1984,20 @@ "ObjectType": "", "ParentTableName": "schema2.mixed_data_types_table1", "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "employees3", + "RowCount": 2, + "ColumnCount": 3, + "Reads": 0, + "Writes": 2, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 } ], "Notes": [ @@ -2182,6 +2197,12 @@ "Query": "SELECT lo_create($1)", "DocsLink": "", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "COPY FROM ... WHERE", + "Query": "COPY employees3 (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": [ @@ -2231,4 +2252,4 @@ "MinimumVersionsFixedIn": null } ] -} \ No newline at end of file +} diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index cc72510395..284c8f7d01 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -138,4 +138,32 @@ FROM ) AS items; -SELECT lo_create('32142'); \ No newline at end of file +SELECT lo_create('32142'); + +-- Unsupported COPY constructs + +CREATE TABLE IF NOT EXISTS employees3 ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + age INT NOT NULL +); + + +-- COPY FROM with WHERE clause +COPY employees3 (id, name, age) +FROM STDIN WITH (FORMAT csv) +WHERE age > 30; +1,John Smith,25 +2,Jane Doe,34 +3,Bob Johnson,31 +\. + +-- This can be uncommented when we start using PG 17 or later in the tests +-- -- COPY with ON_ERROR clause +-- COPY employees (id, name, age) +-- FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE ); +-- 4,Adam Smith,22 +-- 5,John Doe,34 +-- 6,Ron Johnson,31 +-- \. + diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index f0f86239a5..8d319777e3 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -70,6 +70,8 @@ const ( XML_FUNCTIONS_NAME = "XML Functions" REGEX_FUNCTIONS = "REGEX_FUNCTIONS" + COPY_FROM_WHERE = "COPY FROM ... WHERE" + COPY_ON_ERROR = "COPY ... ON_ERROR" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 8a57c581df..e6ec4c4cca 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,6 +16,8 @@ limitations under the License. package queryissue import ( + "slices" + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" @@ -204,6 +206,55 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type CopyCommandUnsupportedConstructsDetector struct { + query string + copyFromWhereConstructDetected bool + copyOnErrorConstructDetected bool +} + +func NewCopyCommandUnsupportedConstructsDetector(query string) *CopyCommandUnsupportedConstructsDetector { + return &CopyCommandUnsupportedConstructsDetector{ + query: query, + } +} + +// Detect if COPY command uses unsupported syntax i.e. COPY FROM ... WHERE and COPY... ON_ERROR +func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) error { + // Check if the message is a COPY statement + if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPYSTSMT_NODE { + return nil // Not a COPY statement, nothing to detect + } + + // Check for COPY FROM ... WHERE clause + fromField := queryparser.GetBoolField(msg, "is_from") + whereField := queryparser.GetMessageField(msg, "where_clause") + if fromField && whereField != nil { + d.copyFromWhereConstructDetected = true + } + + // Check for COPY ... ON_ERROR clause + defNames, err := queryparser.TraverseAndExtractDefNamesFromDefElem(msg) + if err != nil { + log.Errorf("error extracting defnames from COPY statement: %v", err) + } + if slices.Contains(defNames, "on_error") { + d.copyOnErrorConstructDetected = true + } + + return nil +} + +func (d *CopyCommandUnsupportedConstructsDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.copyFromWhereConstructDetected { + issues = append(issues, NewCopyFromWhereIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + if d.copyOnErrorConstructDetected { + issues = append(issues, NewCopyOnErrorIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type JsonConstructorFuncDetector struct { query string unsupportedJsonConstructorFunctionsDetected mapset.Set[string] @@ -253,19 +304,19 @@ func NewJsonQueryFunctionDetector(query string) *JsonQueryFunctionDetector { func (j *JsonQueryFunctionDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_TABLE_NODE { /* - SELECT * FROM json_table( - '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, - '$[*]' - COLUMNS ( - column_a int4 path '$.a', - column_b int4 path '$.b' - ) - ); - stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} - from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} - location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} - columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} - pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... + SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + ); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} + from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} + location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} + columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} + pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... */ j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) return nil diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 43dc8ab07f..e6915177c7 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -141,3 +141,29 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin } return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } + +var copyFromWhereIssue = issue.Issue{ + Type: COPY_FROM_WHERE, + TypeName: "COPY FROM ... WHERE", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(copyFromWhereIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var copyOnErrorIssue = issue.Issue{ + Type: COPY_ON_ERROR, + TypeName: "COPY ... ON_ERROR", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(copyOnErrorIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 5b0cd0151b..d24d4f1947 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -58,6 +58,29 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testCopyOnErrorIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + // In case the COPY ... ON_ERROR construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. + _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: option \"on_error\" not recognized (SQLSTATE 42601)", copyOnErrorIssue) + +} + +func testCopyFromWhereIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + // In case the COPY FROM ... WHERE construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. + _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WHERE loid = 1 WITH (FORMAT csv, HEADER true);`) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: syntax error at or near \"WHERE\" (SQLSTATE 42601)", copyFromWhereIssue) +} + func testJsonConstructorFunctions(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -145,6 +168,11 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "copy on error", ybVersion), testCopyOnErrorIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "copy from where", ybVersion), testCopyFromWhereIssue) + assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "json constructor functions", ybVersion), testJsonConstructorFunctions) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 8b5f028b95..9c32fca533 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), } @@ -414,7 +415,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) xmlIssueAdded = true } } - result = append(result, issues...) + result = append(result, issue) } } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 1450f4edd8..0d6e9d23f4 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -649,3 +649,42 @@ func TestRegexFunctionsIssue(t *testing.T) { } } + +func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { + expectedIssues := map[string][]QueryIssue{ + `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`)}, + `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`)}, + `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`)}, + + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`)}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`)}, + + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`), NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`)}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`), NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`)}, + + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT csv);`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT text);`: {}, + `COPY my_table FROM '/path/to/data.csv';`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (DELIMITER ',');`: {}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true);`: {}, + } + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedIssues { + issues, err := parserIssueDetector.getDMLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, len(expectedIssues), len(issues)) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index e0de006047..8fe62bc82e 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -366,6 +366,14 @@ func GetMessageField(msg protoreflect.Message, fieldName string) protoreflect.Me return nil } +func GetBoolField(msg protoreflect.Message, fieldName string) bool { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).Bool() + } + return false +} + // GetListField retrieves a list field from a message. func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List { field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) @@ -403,12 +411,10 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} - Extract all defnames from the def_eleme node */ func TraverseAndExtractDefNamesFromDefElem(msg protoreflect.Message) ([]string, error) { var defNames []string - collectorFunc := func(msg protoreflect.Message) error { if GetMsgFullName(msg) != PG_QUERY_DEFELEM_NODE { return nil diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index c988ee48eb..e742a81724 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -49,6 +49,7 @@ const ( PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" + PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" ) // function type for processing nodes during traversal From 0351d82d97402e8c0f00af38469269b0c8bfe2ce Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 26 Dec 2024 18:36:06 +0530 Subject: [PATCH 079/105] Reporting multirange datatypes in analyze schema and assessment report for PG(#2112) --- .../dummy-export-dir/schema/tables/table.sql | 36 ++- .../tests/analyze-schema/expected_issues.json | 60 ++++ migtests/tests/analyze-schema/summary.json | 8 +- .../expectedAssessmentReport.json | 267 +++++++++++++++++- .../pg_assessment_report.sql | 30 ++ yb-voyager/src/query/queryissue/constants.go | 6 +- .../src/query/queryissue/detectors_ddl.go | 8 + yb-voyager/src/query/queryissue/issues_ddl.go | 14 + .../src/query/queryissue/issues_ddl_test.go | 40 +++ yb-voyager/src/srcdb/postgres.go | 4 +- 10 files changed, 457 insertions(+), 16 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 82f7411d38..3d9cddc032 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -384,4 +384,38 @@ CREATE TABLE public.locations ( created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP ); -CREATE TABLE image (title text, raster lo); \ No newline at end of file +CREATE TABLE image (title text, raster lo); + +-- create table with multirange data types + +-- Create tables with primary keys directly +CREATE TABLE bigint_multirange_table ( + id integer PRIMARY KEY, + value_ranges int8multirange +); + +CREATE TABLE date_multirange_table ( + id integer PRIMARY KEY, + project_dates datemultirange +); + +CREATE TABLE int_multirange_table ( + id integer PRIMARY KEY, + value_ranges int4multirange +); + +CREATE TABLE numeric_multirange_table ( + id integer PRIMARY KEY, + price_ranges nummultirange +); + +CREATE TABLE timestamp_multirange_table ( + id integer PRIMARY KEY, + event_times tsmultirange +); + +CREATE TABLE timestamptz_multirange_table ( + id integer PRIMARY KEY, + global_event_times tstzmultirange +); + diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 28e11bd8c4..a6a63a62ba 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1930,5 +1930,65 @@ "Suggestion": "Large objects are not yet supported in YugabyteDB, no workaround available currently", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "bigint_multirange_table", + "Reason": "Unsupported datatype - int8multirange on column - value_ranges", + "SqlStatement": "CREATE TABLE bigint_multirange_table (\n id integer PRIMARY KEY,\n value_ranges int8multirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "date_multirange_table", + "Reason": "Unsupported datatype - datemultirange on column - project_dates", + "SqlStatement": "CREATE TABLE date_multirange_table (\n id integer PRIMARY KEY,\n project_dates datemultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "int_multirange_table", + "Reason": "Unsupported datatype - int4multirange on column - value_ranges", + "SqlStatement": "CREATE TABLE int_multirange_table (\n id integer PRIMARY KEY,\n value_ranges int4multirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "numeric_multirange_table", + "Reason": "Unsupported datatype - nummultirange on column - price_ranges", + "SqlStatement": "CREATE TABLE numeric_multirange_table (\n id integer PRIMARY KEY,\n price_ranges nummultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "timestamp_multirange_table", + "Reason": "Unsupported datatype - tsmultirange on column - event_times", + "SqlStatement": "CREATE TABLE timestamp_multirange_table (\n id integer PRIMARY KEY,\n event_times tsmultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "timestamptz_multirange_table", + "Reason": "Unsupported datatype - tstzmultirange on column - global_event_times", + "SqlStatement": "CREATE TABLE timestamptz_multirange_table (\n id integer PRIMARY KEY,\n global_event_times tstzmultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index f2d1304f39..bdc832263d 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 51, - "InvalidCount": 43, - "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, + "TotalCount": 57, + "InvalidCount": 49, + "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, { "ObjectType": "INDEX", "TotalCount": 43, @@ -91,4 +91,4 @@ "ObjectNames": "\u003c%" } ] -} \ No newline at end of file +} diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 5a861d6410..9ac91b5145 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 29, + "TotalCount": 41, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 68, - "InvalidCount": 23, - "ObjectNames": "public.employees3, public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" + "TotalCount": 80, + "InvalidCount": 35, + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -171,9 +171,21 @@ "public.library_nested", "public.orders_lateral", "public.employees", - "public.employees3" + "public.employees3", + "public.bigint_multirange_table", + "public.date_multirange_table", + "public.int_multirange_table", + "public.numeric_multirange_table", + "public.timestamp_multirange_table", + "public.timestamptz_multirange_table", + "schema2.bigint_multirange_table", + "schema2.date_multirange_table", + "schema2.int_multirange_table", + "schema2.numeric_multirange_table", + "schema2.timestamp_multirange_table", + "schema2.timestamptz_multirange_table" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 74 objects (66 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -269,6 +281,78 @@ "TableName": "orders_lateral", "ColumnName": "order_details", "DataType": "xml" + }, + { + "SchemaName": "public", + "TableName": "date_multirange_table", + "ColumnName": "project_dates", + "DataType": "datemultirange" + }, + { + "SchemaName": "public", + "TableName": "numeric_multirange_table", + "ColumnName": "price_ranges", + "DataType": "nummultirange" + }, + { + "SchemaName": "public", + "TableName": "bigint_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int8multirange" + }, + { + "SchemaName": "public", + "TableName": "int_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int4multirange" + }, + { + "SchemaName": "public", + "TableName": "timestamp_multirange_table", + "ColumnName": "event_times", + "DataType": "tsmultirange" + }, + { + "SchemaName": "public", + "TableName": "timestamptz_multirange_table", + "ColumnName": "global_event_times", + "DataType": "tstzmultirange" + }, + { + "SchemaName": "schema2", + "TableName": "date_multirange_table", + "ColumnName": "project_dates", + "DataType": "datemultirange" + }, + { + "SchemaName": "schema2", + "TableName": "bigint_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int8multirange" + }, + { + "SchemaName": "schema2", + "TableName": "int_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int4multirange" + }, + { + "SchemaName": "schema2", + "TableName": "numeric_multirange_table", + "ColumnName": "price_ranges", + "DataType": "nummultirange" + }, + { + "SchemaName": "schema2", + "TableName": "timestamp_multirange_table", + "ColumnName": "event_times", + "DataType": "tsmultirange" + }, + { + "SchemaName": "schema2", + "TableName": "timestamptz_multirange_table", + "ColumnName": "global_event_times", + "DataType": "tstzmultirange" } ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", @@ -1998,7 +2082,176 @@ "ObjectType": "", "ParentTableName": null, "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "bigint_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "date_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "int_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "numeric_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "timestamp_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "timestamptz_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "bigint_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "date_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "int_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "numeric_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "timestamp_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "timestamptz_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 } + ], "Notes": [ "There are some Unlogged tables in the schema. They will be created as regular LOGGED tables in YugabyteDB as unlogged tables are not supported." diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index c23e2ddc5f..e1411914df 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -24,6 +24,36 @@ CREATE TABLE Mixed_Data_Types_Table2 ( path_data PATH ); +CREATE TABLE int_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int4multirange +); + +CREATE TABLE bigint_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int8multirange +); + +CREATE TABLE numeric_multirange_table ( + id SERIAL PRIMARY KEY, + price_ranges nummultirange +); + +CREATE TABLE timestamp_multirange_table ( + id SERIAL PRIMARY KEY, + event_times tsmultirange +); + +CREATE TABLE timestamptz_multirange_table ( + id SERIAL PRIMARY KEY, + global_event_times tstzmultirange +); + +CREATE TABLE date_multirange_table ( + id SERIAL PRIMARY KEY, + project_dates datemultirange +); + -- GIST Index on point_data column CREATE INDEX idx_point_data ON Mixed_Data_Types_Table1 USING GIST (point_data); diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 8d319777e3..329c256586 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -70,8 +70,10 @@ const ( XML_FUNCTIONS_NAME = "XML Functions" REGEX_FUNCTIONS = "REGEX_FUNCTIONS" - COPY_FROM_WHERE = "COPY FROM ... WHERE" - COPY_ON_ERROR = "COPY ... ON_ERROR" + + MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" + COPY_FROM_WHERE = "COPY FROM ... WHERE" + COPY_ON_ERROR = "COPY ... ON_ERROR" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 82ea56d048..595680f9ae 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -236,6 +236,14 @@ func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, obj "", col.ColumnName, )) + case "int8multirange", "int4multirange", "datemultirange", "nummultirange", "tsmultirange", "tstzmultirange": + *issues = append(*issues, NewMultiRangeDatatypeIssue( + objType, + objName, + "", + col.TypeName, + col.ColumnName, + )) default: *issues = append(*issues, NewUnsupportedDatatypesIssue( objType, diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 914dcd5363..ba1d4cdaed 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -440,6 +440,20 @@ func NewLODatatypeIssue(objectType string, objectName string, SqlStatement strin return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } +var multiRangeDatatypeIssue = issue.Issue{ + Type: MULTI_RANGE_DATATYPE, + TypeName: "Unsupported datatype", + Suggestion: "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + GH: "", //TODO + DocsLink: "", //TODO +} + +func NewMultiRangeDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { + issue := multiRangeDatatypeIssue + issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var securityInvokerViewIssue = issue.Issue{ Type: SECURITY_INVOKER_VIEWS, TypeName: "Security Invoker Views not supported yet", diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 68993779aa..46e913bd1b 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -216,6 +216,44 @@ func testLoDatatypeIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", loDatatypeIssue) } +func testMultiRangeDatatypeIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + queries := []string{ + `CREATE TABLE int_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int4multirange + );`, + `CREATE TABLE bigint_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int8multirange + );`, + `CREATE TABLE numeric_multirange_table ( + id SERIAL PRIMARY KEY, + price_ranges nummultirange + );`, + `CREATE TABLE timestamp_multirange_table ( + id SERIAL PRIMARY KEY, + event_times tsmultirange + );`, + `CREATE TABLE timestamptz_multirange_table ( + id SERIAL PRIMARY KEY, + global_event_times tstzmultirange + );`, + `CREATE TABLE date_multirange_table ( + id SERIAL PRIMARY KEY, + project_dates datemultirange + );`, + } + + for _, query := range queries { + _, err = conn.Exec(ctx, query) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", multiRangeDatatypeIssue) + } +} + func testSecurityInvokerView(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -291,6 +329,8 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "lo datatype", ybVersion), testLoDatatypeIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "multi range datatype", ybVersion), testMultiRangeDatatypeIssue) + assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "security invoker view", ybVersion), testSecurityInvokerView) assert.True(t, success) } diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index ba2e21b720..5d7a83813a 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -50,8 +50,8 @@ const PG_STAT_STATEMENTS = "pg_stat_statements" var pg_catalog_tables_required = []string{"regclass", "pg_class", "pg_inherits", "setval", "pg_index", "pg_relation_size", "pg_namespace", "pg_tables", "pg_sequences", "pg_roles", "pg_database", "pg_extension"} var information_schema_tables_required = []string{"schemata", "tables", "columns", "key_column_usage", "sequences"} -var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID", "LO"} -var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "LO"} +var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID", "LO", "INT4MULTIRANGE", "INT8MULTIRANGE", "NUMMULTIRANGE", "TSMULTIRANGE", "TSTZMULTIRANGE", "DATEMULTIRANGE"} +var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "LO", "INT4MULTIRANGE", "INT8MULTIRANGE", "NUMMULTIRANGE", "TSMULTIRANGE", "TSTZMULTIRANGE", "DATEMULTIRANGE"} func GetPGLiveMigrationUnsupportedDatatypes() []string { liveMigrationUnsupportedDataTypes, _ := lo.Difference(PostgresUnsupportedDataTypesForDbzm, PostgresUnsupportedDataTypes) From bbfc84cd40ecbe3a570077181b1281014b77eed1 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Fri, 27 Dec 2024 12:54:27 +0530 Subject: [PATCH 080/105] Adding build tags to each test file (#2116) - introduced three build tags - unit, integration, issues_integration - every new test file added should have one of these tags - any untagged test file will be executed with every go test command --- .github/workflows/go.yml | 2 +- .github/workflows/integration-tests.yml | 2 +- .github/workflows/issue-tests.yml | 2 +- yb-voyager/cmd/analyzeSchema_test.go | 2 ++ .../cmd/assessMigrationBulkCommand_test.go | 2 ++ yb-voyager/cmd/common_test.go | 17 +++++++++++++++++ yb-voyager/cmd/exportDataStatus_test.go | 17 +++++++++++++++++ yb-voyager/cmd/exportSchema_test.go | 5 ++++- yb-voyager/cmd/import_data_test.go | 17 +++++++++++++++++ yb-voyager/cmd/live_migration_test.go | 2 ++ .../adaptive_parallelism_test.go | 2 ++ yb-voyager/src/callhome/diagnostics_test.go | 17 +++++++++++++++++ yb-voyager/src/cp/yugabyted/yugabyted_test.go | 17 +++++++++++++++++ yb-voyager/src/datafile/descriptor_test.go | 18 ++++++++++++++++++ yb-voyager/src/dbzm/status_test.go | 17 +++++++++++++++++ yb-voyager/src/issue/issue_test.go | 3 ++- yb-voyager/src/metadb/metadataDB_test.go | 17 +++++++++++++++++ .../src/migassessment/assessmentDB_test.go | 17 +++++++++++++++++ yb-voyager/src/migassessment/sizing_test.go | 2 ++ yb-voyager/src/namereg/namereg_test.go | 17 +++++++++++++++++ .../src/query/queryissue/detectors_test.go | 2 ++ .../src/query/queryissue/issues_ddl_test.go | 15 ++++++--------- .../src/query/queryissue/issues_dml_test.go | 5 ++++- .../queryissue/parser_issue_detector_test.go | 17 ++++++++++------- .../src/query/queryparser/traversal_test.go | 2 ++ .../src/tgtdb/attr_name_registry_test.go | 17 +++++++++++++++++ yb-voyager/src/tgtdb/conn_pool_test.go | 2 ++ yb-voyager/src/tgtdb/main_test.go | 2 ++ yb-voyager/src/tgtdb/postgres_test.go | 2 ++ .../src/tgtdb/suites/yugabytedbSuite_test.go | 2 ++ yb-voyager/src/tgtdb/yugabytedb_test.go | 2 ++ yb-voyager/src/utils/jsonfile/jsonfile_test.go | 17 +++++++++++++++++ yb-voyager/src/utils/sqlname/nametuple_test.go | 2 ++ yb-voyager/src/utils/struct_map_test.go | 17 +++++++++++++++++ yb-voyager/src/ybversion/yb_version_test.go | 2 ++ yb-voyager/test/utils/testutils.go | 6 ++++++ 36 files changed, 285 insertions(+), 22 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ea81fd4e7c..88c20428d5 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' + go test -v ./... -tags 'unit' - name: Vet run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b6b018be21..d94a594cb1 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Run Integration Tests run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags 'integration' + go test -v ./... -tags 'integration' diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 7db2e6260c..550eef3c8a 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -47,4 +47,4 @@ jobs: - name: Test Issues Against YB Version run: | cd yb-voyager - go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' + go test -v ./... -tags 'issues_integration' diff --git a/yb-voyager/cmd/analyzeSchema_test.go b/yb-voyager/cmd/analyzeSchema_test.go index 1b97697d31..d768b94b06 100644 --- a/yb-voyager/cmd/analyzeSchema_test.go +++ b/yb-voyager/cmd/analyzeSchema_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/cmd/assessMigrationBulkCommand_test.go b/yb-voyager/cmd/assessMigrationBulkCommand_test.go index cfb8b3436f..5b887c901a 100644 --- a/yb-voyager/cmd/assessMigrationBulkCommand_test.go +++ b/yb-voyager/cmd/assessMigrationBulkCommand_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index 6a5c462365..49eee13b08 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package cmd import ( diff --git a/yb-voyager/cmd/exportDataStatus_test.go b/yb-voyager/cmd/exportDataStatus_test.go index 692710aace..20142dba4a 100644 --- a/yb-voyager/cmd/exportDataStatus_test.go +++ b/yb-voyager/cmd/exportDataStatus_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package cmd import ( diff --git a/yb-voyager/cmd/exportSchema_test.go b/yb-voyager/cmd/exportSchema_test.go index 432f763330..a399b0e70a 100644 --- a/yb-voyager/cmd/exportSchema_test.go +++ b/yb-voyager/cmd/exportSchema_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. @@ -17,9 +19,10 @@ limitations under the License. package cmd import ( - "github.com/stretchr/testify/assert" "strings" "testing" + + "github.com/stretchr/testify/assert" ) func TestShardingRecommendations(t *testing.T) { diff --git a/yb-voyager/cmd/import_data_test.go b/yb-voyager/cmd/import_data_test.go index e8c3a2300f..59fccddf30 100644 --- a/yb-voyager/cmd/import_data_test.go +++ b/yb-voyager/cmd/import_data_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package cmd import ( diff --git a/yb-voyager/cmd/live_migration_test.go b/yb-voyager/cmd/live_migration_test.go index 3ace7907a3..81f9da5466 100644 --- a/yb-voyager/cmd/live_migration_test.go +++ b/yb-voyager/cmd/live_migration_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go b/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go index 7b4438d9e9..139b443f4d 100644 --- a/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go +++ b/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index 9d63b562b5..84781c01fa 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package callhome import ( diff --git a/yb-voyager/src/cp/yugabyted/yugabyted_test.go b/yb-voyager/src/cp/yugabyted/yugabyted_test.go index 90bf0c5e25..8e94839d06 100644 --- a/yb-voyager/src/cp/yugabyted/yugabyted_test.go +++ b/yb-voyager/src/cp/yugabyted/yugabyted_test.go @@ -1,3 +1,20 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package yugabyted import ( diff --git a/yb-voyager/src/datafile/descriptor_test.go b/yb-voyager/src/datafile/descriptor_test.go index 6092aba4c3..7c8b5eace3 100644 --- a/yb-voyager/src/datafile/descriptor_test.go +++ b/yb-voyager/src/datafile/descriptor_test.go @@ -1,3 +1,21 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package datafile import ( diff --git a/yb-voyager/src/dbzm/status_test.go b/yb-voyager/src/dbzm/status_test.go index 753188dfd8..cfd1f4deaf 100644 --- a/yb-voyager/src/dbzm/status_test.go +++ b/yb-voyager/src/dbzm/status_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package dbzm import ( diff --git a/yb-voyager/src/issue/issue_test.go b/yb-voyager/src/issue/issue_test.go index 19ae50de6d..4dd58a5649 100644 --- a/yb-voyager/src/issue/issue_test.go +++ b/yb-voyager/src/issue/issue_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. @@ -13,7 +15,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package issue import ( diff --git a/yb-voyager/src/metadb/metadataDB_test.go b/yb-voyager/src/metadb/metadataDB_test.go index b6c0126927..bee70480ea 100644 --- a/yb-voyager/src/metadb/metadataDB_test.go +++ b/yb-voyager/src/metadb/metadataDB_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package metadb import ( diff --git a/yb-voyager/src/migassessment/assessmentDB_test.go b/yb-voyager/src/migassessment/assessmentDB_test.go index 854b52c8fd..204acbb128 100644 --- a/yb-voyager/src/migassessment/assessmentDB_test.go +++ b/yb-voyager/src/migassessment/assessmentDB_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package migassessment import ( diff --git a/yb-voyager/src/migassessment/sizing_test.go b/yb-voyager/src/migassessment/sizing_test.go index dff6c52522..f65bfeb5fb 100644 --- a/yb-voyager/src/migassessment/sizing_test.go +++ b/yb-voyager/src/migassessment/sizing_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index c785dbff02..f4c1430daf 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package namereg import ( diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 266f053ba0..16c136d3bc 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 46e913bd1b..d53896781f 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -1,5 +1,8 @@ +//go:build issues_integration + /* Copyright (c) YugabyteDB, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -24,9 +27,9 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" - "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) var ( @@ -56,15 +59,9 @@ func getConn() (*pgx.Conn, error) { return conn, nil } -func fatalIfError(t *testing.T, err error) { - if err != nil { - t.Fatalf("error: %v", err) - } -} - func assertErrorCorrectlyThrownForIssueForYBVersion(t *testing.T, execErr error, expectedError string, issue issue.Issue) { isFixed, err := issue.IsFixedIn(testYbVersion) - fatalIfError(t, err) + testutils.FatalIfError(t, err) if isFixed { assert.NoError(t, execErr) @@ -285,7 +282,7 @@ func TestDDLIssuesInYBVersion(t *testing.T) { ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] testYbVersion, err = ybversion.NewYBVersion(ybVersionWithoutBuild) - fatalIfError(t, err) + testutils.FatalIfError(t, err) testYugabytedbConnStr = os.Getenv("YB_CONN_STR") if testYugabytedbConnStr == "" { diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index d24d4f1947..8bbda2d349 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -1,3 +1,5 @@ +//go:build issues_integration + /* Copyright (c) YugabyteDB, Inc. Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -146,7 +149,7 @@ func TestDMLIssuesInYBVersion(t *testing.T) { ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] testYbVersion, err = ybversion.NewYBVersion(ybVersionWithoutBuild) - fatalIfError(t, err) + testutils.FatalIfError(t, err) testYugabytedbConnStr = os.Getenv("YB_CONN_STR") if testYugabytedbConnStr == "" { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 0d6e9d23f4..0481f0d354 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. @@ -23,6 +25,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/samber/lo" "github.com/stretchr/testify/assert" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -308,12 +311,12 @@ func TestUnloggedTableIssueReportedInOlderVersion(t *testing.T) { // Not reported by default issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 0, len(issues)) // older version should report the issue issues, err = parserIssueDetector.GetDDLIssues(stmt, ybversion.V2024_1_0_0) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) assert.True(t, cmp.Equal(issues[0], NewUnloggedTableIssue("TABLE", "tbl_unlog", stmt))) } @@ -488,7 +491,7 @@ func TestSingleXMLIssueIsDetected(t *testing.T) { parserIssueDetector := NewParserIssueDetector() issues, err := parserIssueDetector.getDMLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) } @@ -543,7 +546,7 @@ FROM books;`, `SELECT id, JSON_VALUE(details, '$.title') AS title FROM books WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`, -`CREATE MATERIALIZED VIEW public.test_jsonb_view AS + `CREATE MATERIALIZED VIEW public.test_jsonb_view AS SELECT id, data->>'name' AS name, @@ -556,7 +559,7 @@ JSON_TABLE(data, '$.skills[*]' skill TEXT PATH '$' ) ) AS jt;`, - `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, + `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, } sqlsWithExpectedIssues := map[string][]QueryIssue{ sqls[0]: []QueryIssue{ @@ -636,14 +639,14 @@ func TestRegexFunctionsIssue(t *testing.T) { for _, stmt := range dmlStmts { issues, err := parserIssueDetector.getDMLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) assert.Equal(t, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", stmt), issues[0]) } for _, stmt := range ddlStmts { issues, err := parserIssueDetector.getDDLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) assert.Equal(t, NewRegexFunctionsIssue(TABLE_OBJECT_TYPE, "x", stmt), issues[0]) } diff --git a/yb-voyager/src/query/queryparser/traversal_test.go b/yb-voyager/src/query/queryparser/traversal_test.go index 9e0f73d61e..6e65579719 100644 --- a/yb-voyager/src/query/queryparser/traversal_test.go +++ b/yb-voyager/src/query/queryparser/traversal_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/attr_name_registry_test.go b/yb-voyager/src/tgtdb/attr_name_registry_test.go index 8b656336fe..3655e477d0 100644 --- a/yb-voyager/src/tgtdb/attr_name_registry_test.go +++ b/yb-voyager/src/tgtdb/attr_name_registry_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( diff --git a/yb-voyager/src/tgtdb/conn_pool_test.go b/yb-voyager/src/tgtdb/conn_pool_test.go index 7d4ce43805..55053950d1 100644 --- a/yb-voyager/src/tgtdb/conn_pool_test.go +++ b/yb-voyager/src/tgtdb/conn_pool_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/main_test.go b/yb-voyager/src/tgtdb/main_test.go index 46cc2150e2..ea95391485 100644 --- a/yb-voyager/src/tgtdb/main_test.go +++ b/yb-voyager/src/tgtdb/main_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go index e0e9bd60c5..9bb3d563ef 100644 --- a/yb-voyager/src/tgtdb/postgres_test.go +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go index 834263a61e..8f51d621b2 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go index 755ed46d67..e310877a92 100644 --- a/yb-voyager/src/tgtdb/yugabytedb_test.go +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/utils/jsonfile/jsonfile_test.go b/yb-voyager/src/utils/jsonfile/jsonfile_test.go index 8818734334..a0f20b57e0 100644 --- a/yb-voyager/src/utils/jsonfile/jsonfile_test.go +++ b/yb-voyager/src/utils/jsonfile/jsonfile_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package jsonfile import ( diff --git a/yb-voyager/src/utils/sqlname/nametuple_test.go b/yb-voyager/src/utils/sqlname/nametuple_test.go index 5e7dc33ebf..b89ade4c8d 100644 --- a/yb-voyager/src/utils/sqlname/nametuple_test.go +++ b/yb-voyager/src/utils/sqlname/nametuple_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/utils/struct_map_test.go b/yb-voyager/src/utils/struct_map_test.go index 788251706e..c41f90a7d7 100644 --- a/yb-voyager/src/utils/struct_map_test.go +++ b/yb-voyager/src/utils/struct_map_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package utils import ( diff --git a/yb-voyager/src/ybversion/yb_version_test.go b/yb-voyager/src/ybversion/yb_version_test.go index 0e53526751..e5e67b04ef 100644 --- a/yb-voyager/src/ybversion/yb_version_test.go +++ b/yb-voyager/src/ybversion/yb_version_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/test/utils/testutils.go b/yb-voyager/test/utils/testutils.go index 74c30bc446..942e00fd67 100644 --- a/yb-voyager/test/utils/testutils.go +++ b/yb-voyager/test/utils/testutils.go @@ -400,3 +400,9 @@ func WaitForDBToBeReady(db *sql.DB) error { } return fmt.Errorf("database did not become ready in time") } + +func FatalIfError(t *testing.T, err error) { + if err != nil { + t.Fatalf("error: %v", err) + } +} From d148a0475e1e5e63cd9ec435090bdc27f2a3d4a9 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Fri, 27 Dec 2024 14:13:58 +0530 Subject: [PATCH 081/105] Regression: Unit test failing due to recent PR merging using code changed in bbfc84cd40ecbe3a570077181b1281014b77eed1 (#2124) - fatalIfError was renamed to testutils.FatalIfError()in this commit, but recent commit was still using old name and git merge was unable to detect this conflict. --- yb-voyager/src/query/queryissue/parser_issue_detector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 0481f0d354..e4c6f09d13 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -681,7 +681,7 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { for stmt, expectedIssues := range expectedIssues { issues, err := parserIssueDetector.getDMLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, len(expectedIssues), len(issues)) for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { From 47a8e5088c25f0673c85f4813e273eba841acde9 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 27 Dec 2024 15:30:36 +0530 Subject: [PATCH 082/105] Upgrade to PG17 in Github actions (#2114) Fixed Validation files for the VIEW/MVIEW's select statements Skipping the SET transaction_timeout = 0; in the import schema to be not executed on YB. Fixes #2121 --- .github/workflows/misc-migtests.yml | 10 +- .github/workflows/pg-13-migtests.yml | 219 ++++++++++++++++++ .../{pg-migtests.yml => pg-17-migtests.yml} | 20 +- .github/workflows/pg-9-migtests.yml | 6 +- migtests/scripts/postgresql/env.sh | 2 +- migtests/scripts/postgresql/ff_env.sh | 2 +- .../expectedAssessmentReport.json | 14 +- .../expected_schema_analysis_report.json | 14 +- .../expectedAssessmentReport.json | 25 +- .../expectedAssessmentReport.json | 12 +- .../expectedAssessmentReport.json | 2 +- .../expected_files/expected_failed.sql | 30 +-- .../expected_schema_analysis_report.json | 2 +- yb-voyager/cmd/constants.go | 3 + yb-voyager/cmd/importSchemaYugabyteDB.go | 29 +-- 15 files changed, 320 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/pg-13-migtests.yml rename .github/workflows/{pg-migtests.yml => pg-17-migtests.yml} (94%) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index cd9fe7ad35..3385743b8b 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -12,9 +12,9 @@ jobs: services: postgres: - image: postgres:15 + image: postgres:17 env: - POSTGRES_PASSWORD: secret + POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -55,17 +55,19 @@ jobs: sudo apt install -y libpq-dev sudo apt install python3-psycopg2 + #TODO Remove the install PG 17 command once we do that in installer script - name: Run installer script to setup voyager run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support - echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH" + sudo apt-get -y install postgresql-17 + echo "/usr/lib/postgresql/17/bin" >> "$GITHUB_PATH" env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y - name: Test PostgreSQL Connection run: | - psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();" + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" - name: Create PostgreSQL user run: | diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml new file mode 100644 index 0000000000..a64d351fa0 --- /dev/null +++ b/.github/workflows/pg-13-migtests.yml @@ -0,0 +1,219 @@ +name: "PG 13: Migration Tests" + +on: + push: + branches: ['main', '*.*-dev', '*.*.*-dev'] + pull_request: + branches: ['main'] + +jobs: + run-pg-13-migration-tests: + strategy: + matrix: + version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] + BETA_FAST_DATA_EXPORT: [0, 1] + test_group: + - offline + - live_basic + - live_advanced + + env: + BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} + runs-on: ubuntu-22.04 + services: + postgres: + image: postgres:13 + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + # https://github.com/actions/setup-java + with: + distribution: "temurin" + java-version: "17" + check-latest: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: "Enable postgres with wal_level as logical and install postgis" + run: | + docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" + docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3" + docker restart ${{ job.services.postgres.id }} + sleep 10 + + - name: Install python3 and psycopg2 + run: | + sudo apt install -y python3 + sudo apt install -y libpq-dev + sudo apt install python3-psycopg2 + + - name: Run installer script to setup voyager + run: | + cd installer_scripts + yes | ./install-yb-voyager --install-from-local-source --only-pg-support + sudo rm /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo rm /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + pg_dump --version + env: + ON_INSTALLER_ERROR_OUTPUT_LOG: Y + + - name: Test PostgreSQL Connection + run: | + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" + + - name: Create PostgreSQL user + run: | + ./migtests/scripts/postgresql/create_pg_user + + - name: Start YugabyteDB cluster + run: | + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false + sleep 20 + + - name: Test YugabyteDB connection + run: | + psql "postgresql://yugabyte:@127.0.0.1:5433/yugabyte" -c "SELECT version();" + + - name: Create YugabyteDB user + run: | + ./migtests/scripts/yugabytedb/create_yb_user + + - name: Enable yb-tserver-n1 and yb-master-n1 name resolution + run: | + echo "127.0.0.1 yb-tserver-n1" | sudo tee -a /etc/hosts + echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts + psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" + + - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/table-list-flags-tests + + - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh + + - name: "TEST: pg-case-sensitivity-single-table" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table + + - name: "TEST: pg-dvdrental" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/dvdrental + + - name: "TEST: pg-datatypes" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/datatypes + + - name: "TEST: pg-constraints" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/constraints + + - name: "TEST: pg-sequences" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/sequences + + - name: "TEST: pg-indexes" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/indexes + + - name: "TEST: pg-partitions" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/partitions + + - name: "TEST: pg-partitions with (table-list)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions + + # Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529 + # Fixed in 2.17.1.0-b368 + - name: "TEST: pg-partitions-with-indexes" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/partitions-with-indexes + + - name: "TEST: pg-views-and-rules" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/views-and-rules + + - name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/misc-objects-1 + + - name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/misc-objects-2 + + - name: "TEST: pg-dependent-ddls" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/dependent-ddls + + - name: "TEST: pg-multiple-schemas" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/multiple-schemas + + - name: "TEST: pg-codependent-schemas" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/codependent-schemas + + - name: "TEST: pg-sample-schema-emp" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/sample-employee + + - name: "TEST: pg-hasura-ecommerce" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/hasura-ecommerce + + - name: "TEST: pg-case-sensitivity-reserved-words-offline" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words + + - name: "TEST: pg-basic-non-public-live-migration-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test + + - name: "TEST: pg-basic-public-fall-forward-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test + + # - name: "TEST: pg-basic-non-public-fall-back-test" + # run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test + + - name: "TEST: pg-datatypes-fall-back-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes + + # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables. + - name: "TEST: pg-live-migration-multiple-schemas" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas + + - name: "TEST: pg-unique-key-conflicts-test" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test + + - name: "TEST: pg-live-migration-partitions-fall-forward" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions + diff --git a/.github/workflows/pg-migtests.yml b/.github/workflows/pg-17-migtests.yml similarity index 94% rename from .github/workflows/pg-migtests.yml rename to .github/workflows/pg-17-migtests.yml index 0681dd543e..498a4e0cf1 100644 --- a/.github/workflows/pg-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -1,4 +1,4 @@ -name: "PG 13: Migration Tests" +name: "PG 17: Migration Tests" on: push: @@ -7,7 +7,7 @@ on: branches: ['main'] jobs: - run-pg-migration-tests: + run-pg-17-migration-tests: strategy: matrix: version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] @@ -22,9 +22,9 @@ jobs: runs-on: ubuntu-22.04 services: postgres: - image: postgres:13 + image: postgres:17 env: - POSTGRES_PASSWORD: secret + POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -55,7 +55,7 @@ jobs: - name: "Enable postgres with wal_level as logical and install postgis" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" - docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3" + docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-17-postgis postgresql-17-postgis-3" docker restart ${{ job.services.postgres.id }} sleep 10 @@ -65,21 +65,23 @@ jobs: sudo apt install -y libpq-dev sudo apt install python3-psycopg2 + #TODO Remove the install PG 17 command once we do that in installer script - name: Run installer script to setup voyager run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support + sudo apt-get -y install postgresql-17 sudo rm /usr/bin/pg_dump - sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore - sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y - name: Test PostgreSQL Connection run: | - psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();" + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" - name: Create PostgreSQL user run: | @@ -125,7 +127,7 @@ jobs: - name: "TEST: PG sample schemas (pgtbrus)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-schema-migration.sh pg/pgtbrus\ + run: migtests/scripts/run-schema-migration.sh pg/pgtbrus - name: "TEST: PG sample schemas (stackexchange)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index f7b36652fe..ee406af1bc 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -7,7 +7,7 @@ on: branches: ['main'] jobs: - run-pg-migration-tests: + run-pg-9-migration-tests: strategy: matrix: version: [2024.2.0.0-b145] @@ -19,7 +19,7 @@ jobs: postgres: image: postgres:9 env: - POSTGRES_PASSWORD: secret + POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -72,7 +72,7 @@ jobs: - name: Test PostgreSQL Connection run: | - psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();" + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" - name: Create PostgreSQL user run: | diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index ec1f123582..fcc466b822 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -3,4 +3,4 @@ export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} -export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"secret"} +export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} diff --git a/migtests/scripts/postgresql/ff_env.sh b/migtests/scripts/postgresql/ff_env.sh index d8b513f842..0f980435cf 100644 --- a/migtests/scripts/postgresql/ff_env.sh +++ b/migtests/scripts/postgresql/ff_env.sh @@ -2,4 +2,4 @@ export SOURCE_REPLICA_DB_NAME=${SOURCE_REPLICA_DB_NAME:-"ff_db"} export SOURCE_REPLICA_DB_HOST=${SOURCE_REPLICA_DB_HOST:-"127.0.0.1"} export SOURCE_REPLICA_DB_PORT=${SOURCE_REPLICA_DB_PORT:-"5432"} export SOURCE_REPLICA_DB_USER=${SOURCE_REPLICA_DB_USER:-"postgres"} -export SOURCE_REPLICA_DB_PASSWORD=${SOURCE_REPLICA_DB_PASSWORD:-"secret"} \ No newline at end of file +export SOURCE_REPLICA_DB_PASSWORD=${SOURCE_REPLICA_DB_PASSWORD:-"postgres"} \ No newline at end of file diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 90979b4b3f..54dc757994 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -593,15 +593,15 @@ "Objects": [ { "ObjectName": "humanresources.vjobcandidate", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidate.jobcandidateid,\n jobcandidate.businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n jobcandidate.modifieddate\n FROM humanresources.jobcandidate;" + "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidateid,\n businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n modifieddate\n FROM humanresources.jobcandidate;" }, { "ObjectName": "humanresources.vjobcandidateeducation", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jc.jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;" + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;" }, { "ObjectName": "humanresources.vjobcandidateemployment", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidate.jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;" + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;" }, { "ObjectName": "person.vadditionalcontactinfo", @@ -609,19 +609,19 @@ }, { "ObjectName": "production.vproductmodelcatalogdescription", - "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodel.productmodelid,\n productmodel.name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n productmodel.rowguid,\n productmodel.modifieddate\n FROM production.productmodel\n WHERE (productmodel.catalogdescription IS NOT NULL);" + "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodelid,\n name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n rowguid,\n modifieddate\n FROM production.productmodel\n WHERE (catalogdescription IS NOT NULL);" }, { "ObjectName": "production.vproductmodelinstructions", - "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT pm.productmodelid,\n pm.name,\n ((xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, pm.step))[1])::character varying(1024) AS \"Step\",\n pm.rowguid,\n pm.modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;" + "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT productmodelid,\n name,\n ((xpath('/ns:root/text()'::text, instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, step))[1])::character varying(1024) AS \"Step\",\n rowguid,\n modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;" }, { "ObjectName": "sales.vpersondemographics", - "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT person.businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (person.demographics IS NOT NULL);" + "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (demographics IS NOT NULL);" }, { "ObjectName": "sales.vstorewithdemographics", - "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT store.businessentityid,\n store.name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;" + "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT businessentityid,\n name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index fa0a6e7955..c58df24345 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -975,7 +975,7 @@ "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidate", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidate.jobcandidateid,\n jobcandidate.businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n jobcandidate.modifieddate\n FROM humanresources.jobcandidate;", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidateid,\n businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n modifieddate\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -987,7 +987,7 @@ "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidateeducation", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jc.jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -999,7 +999,7 @@ "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidateemployment", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidate.jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1023,7 +1023,7 @@ "ObjectType": "VIEW", "ObjectName": "production.vproductmodelcatalogdescription", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodel.productmodelid,\n productmodel.name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n productmodel.rowguid,\n productmodel.modifieddate\n FROM production.productmodel\n WHERE (productmodel.catalogdescription IS NOT NULL);", + "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodelid,\n name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n rowguid,\n modifieddate\n FROM production.productmodel\n WHERE (catalogdescription IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1035,7 +1035,7 @@ "ObjectType": "VIEW", "ObjectName": "production.vproductmodelinstructions", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT pm.productmodelid,\n pm.name,\n ((xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, pm.step))[1])::character varying(1024) AS \"Step\",\n pm.rowguid,\n pm.modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;", + "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT productmodelid,\n name,\n ((xpath('/ns:root/text()'::text, instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, step))[1])::character varying(1024) AS \"Step\",\n rowguid,\n modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1047,7 +1047,7 @@ "ObjectType": "VIEW", "ObjectName": "sales.vpersondemographics", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT person.businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (person.demographics IS NOT NULL);", + "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (demographics IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1059,7 +1059,7 @@ "ObjectType": "VIEW", "ObjectName": "sales.vstorewithdemographics", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT store.businessentityid,\n store.name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;", + "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT businessentityid,\n name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index bbcf4613db..d02458c005 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -28,6 +28,12 @@ "TotalCount": 2, "InvalidCount": 0, "ObjectNames": "analytics.metrics, sales.orders" + }, + { + "ObjectType": "VIEW", + "TotalCount": 1, + "InvalidCount": 1, + "ObjectNames": "sales.employ_depart_view" } ] }, @@ -51,7 +57,18 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": null, + "UnsupportedFeatures": [ + { + "FeatureName": "Aggregate Functions", + "Objects": [ + { + "ObjectName": "sales.employ_depart_view", + "SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n FROM public.employees;" + } + ], + "MinimumVersionsFixedIn": null + } + ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { @@ -91,6 +108,12 @@ "Query": "SELECT metric_name, pg_advisory_lock(metric_id)\nFROM analytics.metrics\nWHERE metric_value \u003e $1", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Aggregate Functions", + "Query": "SELECT\n any_value(name) AS any_employee\n FROM employees", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": null diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 9ac91b5145..ec45f22bf9 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -489,11 +489,11 @@ "Objects": [ { "ObjectName": "public.sales_employees", - "SqlStatement": "CREATE VIEW public.sales_employees AS\n SELECT employees2.id,\n employees2.first_name,\n employees2.last_name,\n employees2.full_name\n FROM public.employees2\n WHERE ((employees2.department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" + "SqlStatement": "CREATE VIEW public.sales_employees AS\n SELECT id,\n first_name,\n last_name,\n full_name\n FROM public.employees2\n WHERE ((department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" }, { "ObjectName": "schema2.sales_employees", - "SqlStatement": "CREATE VIEW schema2.sales_employees AS\n SELECT employees2.id,\n employees2.first_name,\n employees2.last_name,\n employees2.full_name\n FROM schema2.employees2\n WHERE ((employees2.department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" + "SqlStatement": "CREATE VIEW schema2.sales_employees AS\n SELECT id,\n first_name,\n last_name,\n full_name\n FROM schema2.employees2\n WHERE ((department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", @@ -594,7 +594,7 @@ "Objects": [ { "ObjectName": "public.ordersentry_view", - "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT order_id,\n customer_name,\n product_name,\n quantity,\n price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name), XMLELEMENT(NAME \"Quantity\", quantity), XMLELEMENT(NAME \"TotalPrice\", (price * (quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((customer_name || product_name)))::bigint) AS lock_acquired,\n ctid AS row_ctid,\n xmin AS transaction_id\n FROM public.ordersentry;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", @@ -605,7 +605,7 @@ "Objects": [ { "ObjectName": "public.ordersentry_view", - "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT order_id,\n customer_name,\n product_name,\n quantity,\n price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name), XMLELEMENT(NAME \"Quantity\", quantity), XMLELEMENT(NAME \"TotalPrice\", (price * (quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((customer_name || product_name)))::bigint) AS lock_acquired,\n ctid AS row_ctid,\n xmin AS transaction_id\n FROM public.ordersentry;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", @@ -616,7 +616,7 @@ "Objects": [ { "ObjectName": "public.ordersentry_view", - "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT order_id,\n customer_name,\n product_name,\n quantity,\n price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name), XMLELEMENT(NAME \"Quantity\", quantity), XMLELEMENT(NAME \"TotalPrice\", (price * (quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((customer_name || product_name)))::bigint) AS lock_acquired,\n ctid AS row_ctid,\n xmin AS transaction_id\n FROM public.ordersentry;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", @@ -647,7 +647,7 @@ "Objects": [ { "ObjectName": "public.view_explicit_security_invoker", - "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employees.employee_id,\n employees.first_name\n FROM public.employees;" + "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employee_id,\n first_name\n FROM public.employees;" } ], "MinimumVersionsFixedIn": null diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index dd49b1dc67..8cd48f94b6 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -577,7 +577,7 @@ "Objects": [ { "ObjectName": "regress_rls_schema.bv1", - "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT b1.a,\n b1.b\n FROM regress_rls_schema.b1\n WHERE (b1.a \u003e 0)\n WITH CASCADED CHECK OPTION;" + "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT a,\n b\n FROM regress_rls_schema.b1\n WHERE (a \u003e 0)\n WITH CASCADED CHECK OPTION;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", diff --git a/migtests/tests/pg/omnibus/expected_files/expected_failed.sql b/migtests/tests/pg/omnibus/expected_files/expected_failed.sql index cc663a37c2..ad72d68083 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_failed.sql +++ b/migtests/tests/pg/omnibus/expected_files/expected_failed.sql @@ -105,10 +105,10 @@ ERROR: VIEW WITH CASCADED CHECK OPTION not supported yet (SQLSTATE 0A000) File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS - SELECT b1.a, - b1.b + SELECT a, + b FROM regress_rls_schema.b1 - WHERE (b1.a > 0) + WHERE (a > 0) WITH CASCADED CHECK OPTION; /* @@ -223,10 +223,10 @@ ERROR: relation "composite_type_examples.ordinary_table" does not exist (SQLSTAT File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW composite_type_examples.basic_view AS - SELECT ordinary_table.basic_, - ordinary_table._basic, - ordinary_table.nested, - ordinary_table._nested + SELECT basic_, + _basic, + nested, + _nested FROM composite_type_examples.ordinary_table; /* @@ -234,8 +234,8 @@ ERROR: relation "enum_example.bugs" does not exist (SQLSTATE 42P01) File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW enum_example._bugs AS - SELECT bugs.id, - bugs.status + SELECT id, + status FROM enum_example.bugs; /* @@ -243,11 +243,11 @@ ERROR: relation "foreign_db_example.technically_doesnt_exist" does not exist (SQ File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW public.foreign_db_example AS - SELECT technically_doesnt_exist.id, - technically_doesnt_exist.uses_type, - technically_doesnt_exist._uses_type, - technically_doesnt_exist.positive_number, - technically_doesnt_exist._positive_number + SELECT id, + uses_type, + _uses_type, + positive_number, + _positive_number FROM foreign_db_example.technically_doesnt_exist; /* @@ -255,7 +255,7 @@ ERROR: relation "range_type_example.example_tbl" does not exist (SQLSTATE 42P01) File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW range_type_example.depends_on_col_using_type AS - SELECT example_tbl.col + SELECT col FROM range_type_example.example_tbl; /* diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index c5a4f343ef..a2eccb31e1 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -338,7 +338,7 @@ "ObjectType": "VIEW", "ObjectName": "regress_rls_schema.bv1", "Reason": "Schema containing VIEW WITH CHECK OPTION is not supported yet.", - "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT b1.a,\n b1.b\n FROM regress_rls_schema.b1\n WHERE (b1.a \u003e 0)\n WITH CASCADED CHECK OPTION;", + "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT a,\n b\n FROM regress_rls_schema.b1\n WHERE (a \u003e 0)\n WITH CASCADED CHECK OPTION;", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql", "Suggestion": "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716", diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index b22891ae06..e825bc9976 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -93,6 +93,9 @@ const ( CUTOVER_TO_SOURCE = "cutover-to-source" CUTOVER_TO_SOURCE_REPLICA = "cutover-to-source-replica" + CLIENT_MESSAGES_SESSION_VAR = "SET CLIENT_MIN_MESSAGES" + TRANSACTION_TIMEOUT_SESSION_VAR = "SET TRANSACTION_TIMEOUT" + // unsupported features of assess migration VIRTUAL_COLUMN = "VIRTUAL COLUMN" INHERITED_TYPE = "INHERITED TYPE" diff --git a/yb-voyager/cmd/importSchemaYugabyteDB.go b/yb-voyager/cmd/importSchemaYugabyteDB.go index 9cb37b96d9..79720d1477 100644 --- a/yb-voyager/cmd/importSchemaYugabyteDB.go +++ b/yb-voyager/cmd/importSchemaYugabyteDB.go @@ -86,7 +86,6 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo }() sqlInfoArr := parseSqlFileForObjectType(file, objType) - var err error for _, sqlInfo := range sqlInfoArr { if conn == nil { conn = newTargetConn() @@ -97,17 +96,14 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo if !setOrSelectStmt && skipFn != nil && skipFn(objType, sqlInfo.stmt) { continue } - - if objType == "TABLE" { - // Check if the statement should be skipped - skip, err := shouldSkipDDL(sqlInfo.stmt) - if err != nil { - return fmt.Errorf("error checking whether to skip DDL for statement [%s]: %v", sqlInfo.stmt, err) - } - if skip { - log.Infof("Skipping DDL: %s", sqlInfo.stmt) - continue - } + // Check if the statement should be skipped + skip, err := shouldSkipDDL(sqlInfo.stmt, objType) + if err != nil { + return fmt.Errorf("error checking whether to skip DDL for statement [%s]: %v", sqlInfo.stmt, err) + } + if skip { + log.Infof("Skipping DDL: %s", sqlInfo.stmt) + continue } err = executeSqlStmtWithRetries(&conn, sqlInfo, objType) @@ -118,14 +114,19 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo return nil } -func shouldSkipDDL(stmt string) (bool, error) { +func shouldSkipDDL(stmt string, objType string) (bool, error) { stmt = strings.ToUpper(stmt) // pg_dump generate `SET client_min_messages = 'warning';`, but we want to get // NOTICE severity as well (which is the default), hence skipping this. - if strings.Contains(stmt, "SET CLIENT_MIN_MESSAGES") { + //pg_dump 17 gives this SET transaction_timeout = 0; + if strings.Contains(stmt, CLIENT_MESSAGES_SESSION_VAR) || strings.Contains(stmt, TRANSACTION_TIMEOUT_SESSION_VAR) { return true, nil } + if objType != TABLE { + return false, nil + } + skipReplicaIdentity := strings.Contains(stmt, "ALTER TABLE") && strings.Contains(stmt, "REPLICA IDENTITY") if skipReplicaIdentity { return true, nil From a85c6fcbfd0caf72cc271ab27ae4fc800326da01 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 27 Dec 2024 20:30:06 +0530 Subject: [PATCH 083/105] Fix: retrieving migration uuid early in the export schema phase (#2128) --- yb-voyager/cmd/exportSchema.go | 15 +++++++-------- yb-voyager/cmd/importData.go | 9 +++++---- yb-voyager/cmd/importDataFileCommand.go | 4 ++++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index 092a87c39e..22de944ab8 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -90,10 +90,15 @@ func exportSchema() error { utils.PrintAndLog("Schema is not exported yet. Ignoring --start-clean flag.\n\n") } CreateMigrationProjectIfNotExists(source.DBType, exportDir) - + err := retrieveMigrationUUID() + if err != nil { + log.Errorf("failed to get migration UUID: %v", err) + return fmt.Errorf("failed to get migration UUID: %w", err) + } + utils.PrintAndLog("export of schema for source type as '%s'\n", source.DBType) // Check connection with source database. - err := source.DB().Connect() + err = source.DB().Connect() if err != nil { log.Errorf("failed to connect to the source db: %s", err) return fmt.Errorf("failed to connect to the source db: %w", err) @@ -153,12 +158,6 @@ func exportSchema() error { } } - err = retrieveMigrationUUID() - if err != nil { - log.Errorf("failed to get migration UUID: %v", err) - return fmt.Errorf("failed to get migration UUID: %w", err) - } - exportSchemaStartEvent := createExportSchemaStartedEvent() controlPlane.ExportSchemaStarted(&exportSchemaStartEvent) diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index f70d489198..99e71c969b 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -132,6 +132,11 @@ func importDataCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("Failed to get migration status record: %s", err) } + err = retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + // Check if target DB has the required permissions if tconf.RunGuardrailsChecks { checkImportDataPermissions() @@ -470,10 +475,6 @@ func updateTargetConfInMigrationStatus() { } func importData(importFileTasks []*ImportFileTask) { - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } if (importerRole == TARGET_DB_IMPORTER_ROLE || importerRole == IMPORT_FILE_ROLE) && (tconf.EnableUpsert) { if !utils.AskPrompt(color.RedString("WARNING: Ensure that tables on target YugabyteDB do not have secondary indexes. " + diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 5985f4066c..0deaee34b0 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -91,6 +91,10 @@ var importDataFileCmd = &cobra.Command{ dataStore = datastore.NewDataStore(dataDir) importFileTasks := prepareImportFileTasks() prepareForImportDataCmd(importFileTasks) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } importData(importFileTasks) packAndSendImportDataFilePayload(COMPLETE) From fa542b755d018ba7258a4f39e05a5fc7c1d45fee Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 30 Dec 2024 10:15:24 +0530 Subject: [PATCH 084/105] FETCH .. WITH TIES issue detection (#2111) * Added new detector SelectStmtDetector --- .../dummy-export-dir/schema/tables/table.sql | 1 + .../dummy-export-dir/schema/views/view.sql | 7 +- .../tests/analyze-schema/expected_issues.json | 10 +++ migtests/tests/analyze-schema/summary.json | 11 ++-- .../expectedAssessmentReport.json | 64 ++++++++++++++++--- .../pg_assessment_report.sql | 14 ++++ yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/cmd/constants.go | 1 + yb-voyager/go.mod | 3 - yb-voyager/go.sum | 5 -- yb-voyager/src/query/queryissue/constants.go | 4 +- yb-voyager/src/query/queryissue/detectors.go | 34 ++++++++++ yb-voyager/src/query/queryissue/issues_dml.go | 13 ++++ .../src/query/queryissue/issues_dml_test.go | 24 ++++++- .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 63 ++++++++++++++++++ .../src/query/queryparser/helpers_protomsg.go | 15 ++++- .../src/query/queryparser/helpers_struct.go | 4 ++ .../src/query/queryparser/traversal_proto.go | 1 + 19 files changed, 248 insertions(+), 28 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 3d9cddc032..4da2e5f444 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -386,6 +386,7 @@ CREATE TABLE public.locations ( CREATE TABLE image (title text, raster lo); +CREATE TABLE employees (id INT PRIMARY KEY, salary INT); -- create table with multirange data types -- Create tables with primary keys directly diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index 157d311ceb..1fc6a82cbb 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -34,6 +34,11 @@ CREATE VIEW public.orders_view AS orders.xmin AS transaction_id FROM public.orders; +CREATE VIEW top_employees_view AS SELECT * FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees; CREATE VIEW public.my_films_view AS SELECT jt.* FROM my_films, @@ -43,4 +48,4 @@ SELECT jt.* FROM kind text PATH '$.kind', NESTED PATH '$.films[*]' COLUMNS ( title text FORMAT JSON PATH '$.title' OMIT QUOTES, - director text PATH '$.director' KEEP QUOTES))) AS jt; \ No newline at end of file + director text PATH '$.director' KEEP QUOTES))) AS jt; diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index a6a63a62ba..0bfd66b0e5 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1931,6 +1931,16 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "top_employees_view", + "Reason": "FETCH .. WITH TIES", + "SqlStatement": "CREATE VIEW top_employees_view AS SELECT * FROM (\n\t\t\tSELECT * FROM employees\n\t\t\tORDER BY salary DESC\n\t\t\tFETCH FIRST 2 ROWS WITH TIES\n\t\t) AS top_employees;", + "Suggestion": "No workaround available right now", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_datatypes", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index bdc832263d..5e9353c7ac 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,10 @@ }, { "ObjectType": "TABLE", - "TotalCount": 57, + "TotalCount": 58, "InvalidCount": 49, - "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + { "ObjectType": "INDEX", "TotalCount": 43, @@ -50,9 +51,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 6, - "InvalidCount": 6, - "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name" + "TotalCount": 7, + "InvalidCount": 7, + "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name, top_employees_view" }, { "ObjectType": "TRIGGER", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index ec45f22bf9..76e5ffb549 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 41, + "TotalCount": 43, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 80, + "TotalCount": 82, "InvalidCount": 35, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -73,9 +73,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 8, - "InvalidCount": 4, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker" + "TotalCount": 10, + "InvalidCount": 6, + "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view" }, { "ObjectType": "TRIGGER", @@ -162,6 +162,7 @@ "schema2.products", "schema2.foo", "schema2.Case_Sensitive_Columns", + "schema2.employeesforview", "schema2.with_example1", "test_views.xyz_mview", "test_views.view_table2", @@ -183,9 +184,10 @@ "schema2.int_multirange_table", "schema2.numeric_multirange_table", "schema2.timestamp_multirange_table", - "schema2.timestamptz_multirange_table" + "schema2.timestamptz_multirange_table", + "public.employeesforview" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -642,6 +644,20 @@ ], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "FETCH .. WITH TIES Clause", + "Objects": [ + { + "ObjectName": "public.top_employees_view", + "SqlStatement": "CREATE VIEW public.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM public.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" + }, + { + "ObjectName": "schema2.top_employees_view", + "SqlStatement": "CREATE VIEW schema2.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM schema2.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" + } + ], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Security Invoker Views", "Objects": [ @@ -938,7 +954,7 @@ { "SchemaName": "public", "ObjectName": "employees", - "RowCount": 5, + "RowCount": 10, "ColumnCount": 4, "Reads": 0, "Writes": 10, @@ -2069,6 +2085,20 @@ "ParentTableName": "schema2.mixed_data_types_table1", "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "employeesforview", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "public", "ObjectName": "employees3", @@ -2167,6 +2197,20 @@ "ParentTableName": null, "SizeInBytes": 0 }, + { + "SchemaName": "schema2", + "ObjectName": "employeesforview", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "schema2", "ObjectName": "bigint_multirange_table", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index e1411914df..7e3c798ead 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -393,6 +393,20 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- for FETCH .. WITH TIES +CREATE TABLE employeesForView ( + id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + salary NUMERIC(10, 2) NOT NULL +); + +CREATE VIEW top_employees_view AS SELECT * FROM ( + SELECT * FROM employeesForView + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees; + -- SECURITY INVOKER VIEW CREATE TABLE public.employees ( employee_id SERIAL PRIMARY KEY, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 12cae929bd..c54e998f56 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1023,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index e825bc9976..a56ec10970 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -217,6 +217,7 @@ const ( BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables" PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes" REGEX_FUNCTIONS_FEATURE = "Regex Functions" + FETCH_WITH_TIES_FEATURE = "FETCH .. WITH TIES Clause" // Migration caveats diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 8cf7a703f2..7df52aa40d 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -15,7 +15,6 @@ require ( github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 - github.com/fergusstrange/embedded-postgres v1.29.0 github.com/go-sql-driver/mysql v1.7.0 github.com/godror/godror v0.30.2 github.com/google/uuid v1.6.0 @@ -25,7 +24,6 @@ require ( github.com/jackc/pgconn v1.14.3 github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.0.3 - github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 github.com/mitchellh/go-ps v1.0.0 @@ -83,7 +81,6 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index e68839fdc2..1c06c76f90 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -919,8 +919,6 @@ github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fergusstrange/embedded-postgres v1.29.0 h1:Uv8hdhoiaNMuH0w8UuGXDHr60VoAQPFdgx7Qf3bzXJM= -github.com/fergusstrange/embedded-postgres v1.29.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -1971,8 +1969,6 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -2103,7 +2099,6 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 329c256586..586957bf92 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -68,8 +68,8 @@ const ( ADVISORY_LOCKS_NAME = "Advisory Locks" SYSTEM_COLUMNS_NAME = "System Columns" XML_FUNCTIONS_NAME = "XML Functions" - - REGEX_FUNCTIONS = "REGEX_FUNCTIONS" + FETCH_WITH_TIES = "FETCH_WITH_TIES" + REGEX_FUNCTIONS = "REGEX_FUNCTIONS" MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" COPY_FROM_WHERE = "COPY FROM ... WHERE" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index e6ec4c4cca..c834fa6db0 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -206,6 +206,40 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type SelectStmtDetector struct { + query string + limitOptionWithTiesDetected bool +} + +func NewSelectStmtDetector(query string) *SelectStmtDetector { + return &SelectStmtDetector{ + query: query, + } +} + +func (d *SelectStmtDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_SELECTSTMT_NODE { + selectStmtNode, err := queryparser.ProtoAsSelectStmt(msg) + if err != nil { + return err + } + // checks if a SelectStmt node uses a FETCH clause with TIES + // https://www.postgresql.org/docs/13/sql-select.html#SQL-LIMIT + if selectStmtNode.LimitOption == queryparser.LIMIT_OPTION_WITH_TIES { + d.limitOptionWithTiesDetected = true + } + } + return nil +} + +func (d *SelectStmtDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.limitOptionWithTiesDetected { + issues = append(issues, NewFetchWithTiesIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type CopyCommandUnsupportedConstructsDetector struct { query string copyFromWhereConstructDetected bool diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index e6915177c7..7e6f8aed88 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -167,3 +167,16 @@ var copyOnErrorIssue = issue.Issue{ func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { return newQueryIssue(copyOnErrorIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } + +var fetchWithTiesIssue = issue.Issue{ + Type: FETCH_WITH_TIES, + TypeName: "FETCH .. WITH TIES", + TypeDescription: "FETCH .. WITH TIES is not supported in YugabyteDB", + Suggestion: "No workaround available right now", + GH: "", + DocsLink: "", //TODO +} + +func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(fetchWithTiesIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 8bbda2d349..cff5910c26 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -61,16 +61,35 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testFetchWithTiesIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + + stmts := []string{ + `SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES;`, + } + + for _, stmt := range stmts { + _, err = conn.Exec(ctx, stmt) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "WITH"`, regexFunctionsIssue) + } +} + func testCopyOnErrorIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() assert.NoError(t, err) defer conn.Close(context.Background()) + // In case the COPY ... ON_ERROR construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`) assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: option \"on_error\" not recognized (SQLSTATE 42601)", copyOnErrorIssue) - } func testCopyFromWhereIssue(t *testing.T) { @@ -171,6 +190,9 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "fetch with ties", ybVersion), testFetchWithTiesIssue) + assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "copy on error", ybVersion), testCopyOnErrorIssue) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 9c32fca533..c7faf12092 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewSelectStmtDetector(query), NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e4c6f09d13..e5b6315b0c 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -653,6 +653,68 @@ func TestRegexFunctionsIssue(t *testing.T) { } +func TestFetchWithTiesInSelect(t *testing.T) { + + stmt1 := ` + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES;` + + // subquery + stmt2 := `SELECT * + FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees;` + + stmt3 := `CREATE VIEW top_employees_view AS + SELECT * + FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees;` + + expectedIssues := map[string][]QueryIssue{ + stmt1: []QueryIssue{NewFetchWithTiesIssue("DML_QUERY", "", stmt1)}, + stmt2: []QueryIssue{NewFetchWithTiesIssue("DML_QUERY", "", stmt2)}, + } + expectedDDLIssues := map[string][]QueryIssue{ + stmt3: []QueryIssue{NewFetchWithTiesIssue("VIEW", "top_employees_view", stmt3)}, + } + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedIssues { + issues, err := parserIssueDetector.GetDMLIssues(stmt, ybversion.LatestStable) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } + + for stmt, expectedIssues := range expectedDDLIssues { + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} + func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { expectedIssues := map[string][]QueryIssue{ `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, @@ -683,6 +745,7 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { issues, err := parserIssueDetector.getDMLIssues(stmt) testutils.FatalIfError(t, err) assert.Equal(t, len(expectedIssues), len(issues)) + for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { return cmp.Equal(expectedIssue, queryIssue) diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 8fe62bc82e..de3204f3c9 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -21,6 +21,7 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -385,7 +386,7 @@ func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List // GetEnumNumField retrieves a enum field from a message // FieldDescriptor{Syntax: proto3, FullName: pg_query.JsonFuncExpr.op, Number: 1, Cardinality: optional, Kind: enum, HasJSONName: true, JSONName: "op", Enum: pg_query.JsonExprOp} -//val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT +// val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.EnumNumber { field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) if field != nil && msg.Has(field) { @@ -407,6 +408,18 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { return schemaName, objectName } +func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) { + protoMsg, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + selectStmtNode, ok := protoMsg.(*pg_query.SelectStmt) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_SELECTSTMT_NODE) + } + return selectStmtNode, nil +} + /* Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index eafc907910..37d421bdf9 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -23,6 +23,10 @@ import ( "github.com/samber/lo" ) +const ( + LIMIT_OPTION_WITH_TIES = pg_query.LimitOption_LIMIT_OPTION_WITH_TIES +) + func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index e742a81724..9ca3ae3adb 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -41,6 +41,7 @@ const ( PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" + PG_QUERY_SELECTSTMT_NODE = "pg_query.SelectStmt" PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" From 29b09a8b9f6e961e0a7eb4bbd1a72ce217c3bd43 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Fri, 3 Jan 2025 17:20:22 +0530 Subject: [PATCH 085/105] Shipping pg17 in the installer script and consequently in the docker installation too (#2133) --- .github/workflows/pg-13-migtests.yml | 10 ++++------ .github/workflows/pg-17-migtests.yml | 4 ++-- .github/workflows/pg-9-migtests.yml | 8 +++++--- installer_scripts/install-yb-voyager | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index a64d351fa0..d6b03ec165 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -70,10 +70,12 @@ jobs: cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support sudo rm /usr/bin/pg_dump - sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore - sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version + pg_restore --version + psql --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y @@ -119,10 +121,6 @@ jobs: if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - - name: "TEST: pg-dvdrental" - if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-test.sh pg/dvdrental - - name: "TEST: pg-datatypes" if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/datatypes diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index 498a4e0cf1..fc2aba70b3 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -65,17 +65,17 @@ jobs: sudo apt install -y libpq-dev sudo apt install python3-psycopg2 - #TODO Remove the install PG 17 command once we do that in installer script - name: Run installer script to setup voyager run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support - sudo apt-get -y install postgresql-17 sudo rm /usr/bin/pg_dump sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version + pg_restore --version + psql --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index ee406af1bc..8fa3251d1b 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -63,10 +63,12 @@ jobs: cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support sudo rm /usr/bin/pg_dump - sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore - sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version + pg_restore --version + psql --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y @@ -110,4 +112,4 @@ jobs: - name: "TEST: pg-constraints" if: ${{ !cancelled() }} - run: migtests/scripts/run-test.sh pg/constraints \ No newline at end of file + run: migtests/scripts/run-test.sh pg/constraints diff --git a/installer_scripts/install-yb-voyager b/installer_scripts/install-yb-voyager index 912f3e0697..44a49c462f 100755 --- a/installer_scripts/install-yb-voyager +++ b/installer_scripts/install-yb-voyager @@ -142,7 +142,7 @@ centos_main() { output "Installing RPM dependencies." $YUM_INSTALL which wget git gcc make 1>&2 $YUM_INSTALL https://download.postgresql.org/pub/repos/yum/reporpms/EL-${majorVersion}-x86_64/pgdg-redhat-repo-latest.noarch.rpm 1>&2 || true - $YUM_INSTALL postgresql16 1>&2 + $YUM_INSTALL postgresql17 1>&2 $YUM_INSTALL sqlite 1>&2 create_guardrail_scripts_dir create_pg_dump_args_file @@ -953,7 +953,7 @@ ubuntu_install_postgres() { sudo apt install -y postgresql-common 1>&2 echo | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh 1>&2 sudo apt-get update 1>&2 - sudo apt-get -y install postgresql-16 1>&2 + sudo apt-get -y install postgresql-17 1>&2 output "Postgres Installed." } From b7f29af6e1e4d15461f358f221490a219843c952 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 6 Jan 2025 10:27:01 +0530 Subject: [PATCH 086/105] Make voyager import-data metadata tables columns as TEXT (#2130) Currently, voyager metadata tables have columns of type VARCHAR(250). In one of our tests, the table name + export-dir turned out to be longer than 250, thereby leading to data_file_name field exceeding the 250 limit and therefore preventing import-data from running successfully. There's no real reason to limit any of the fields to 250. There isn't any documented difference in performance between the VARCHAR(n) and TEXT, even in postgres Going with ALTER TABLE ALTER COLUMN TYPE so as to ensure that the type is changed even when a user is using voyager on a cluster where voyager was previously used. ALTER table from VARCHAR to TEXT is a safe operation. It does not lead to a table-rewrite and is an idempotent operation. --- yb-voyager/src/tgtdb/postgres.go | 4 ++++ yb-voyager/src/tgtdb/postgres_test.go | 8 ++++---- yb-voyager/src/tgtdb/yugabytedb.go | 4 ++++ yb-voyager/src/tgtdb/yugabytedb_test.go | 8 ++++---- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/yb-voyager/src/tgtdb/postgres.go b/yb-voyager/src/tgtdb/postgres.go index 900252e6db..cb1dfcd217 100644 --- a/yb-voyager/src/tgtdb/postgres.go +++ b/yb-voyager/src/tgtdb/postgres.go @@ -275,6 +275,9 @@ func (pg *TargetPostgreSQL) CreateVoyagerSchema() error { rows_imported BIGINT, PRIMARY KEY (migration_uuid, data_file_name, batch_number, schema_name, table_name) );`, BATCH_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN data_file_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN schema_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME), fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( migration_uuid uuid, channel_no INT, @@ -292,6 +295,7 @@ func (pg *TargetPostgreSQL) CreateVoyagerSchema() error { num_deletes BIGINT, num_updates BIGINT, PRIMARY KEY (migration_uuid, table_name, channel_no));`, EVENTS_PER_TABLE_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, EVENTS_PER_TABLE_METADATA_TABLE_NAME), } maxAttempts := 12 diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go index 9bb3d563ef..9f820ce973 100644 --- a/yb-voyager/src/tgtdb/postgres_test.go +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -50,10 +50,10 @@ func TestCreateVoyagerSchemaPG(t *testing.T) { expectedTables := map[string]map[string]testutils.ColumnPropertiesPG{ BATCH_METADATA_TABLE_NAME: { "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "data_file_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "data_file_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "batch_number": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "schema_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "schema_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "rows_imported": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, }, EVENT_CHANNELS_METADATA_TABLE_NAME: { @@ -66,7 +66,7 @@ func TestCreateVoyagerSchemaPG(t *testing.T) { }, EVENTS_PER_TABLE_METADATA_TABLE_NAME: { "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "channel_no": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "total_events": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, "num_inserts": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index d4f701b4bc..cacf64120c 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -343,6 +343,9 @@ func (yb *TargetYugabyteDB) CreateVoyagerSchema() error { rows_imported BIGINT, PRIMARY KEY (migration_uuid, data_file_name, batch_number, schema_name, table_name) );`, BATCH_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN data_file_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN schema_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME), fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( migration_uuid uuid, channel_no INT, @@ -360,6 +363,7 @@ func (yb *TargetYugabyteDB) CreateVoyagerSchema() error { num_deletes BIGINT, num_updates BIGINT, PRIMARY KEY (migration_uuid, table_name, channel_no));`, EVENTS_PER_TABLE_METADATA_TABLE_NAME), + fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, EVENTS_PER_TABLE_METADATA_TABLE_NAME), } maxAttempts := 12 diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go index e310877a92..33ed248297 100644 --- a/yb-voyager/src/tgtdb/yugabytedb_test.go +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -51,10 +51,10 @@ func TestCreateVoyagerSchemaYB(t *testing.T) { expectedTables := map[string]map[string]testutils.ColumnPropertiesPG{ BATCH_METADATA_TABLE_NAME: { "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "data_file_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "data_file_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "batch_number": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "schema_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "schema_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "rows_imported": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, }, EVENT_CHANNELS_METADATA_TABLE_NAME: { @@ -67,7 +67,7 @@ func TestCreateVoyagerSchemaYB(t *testing.T) { }, EVENTS_PER_TABLE_METADATA_TABLE_NAME: { "migration_uuid": {Type: "uuid", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, - "table_name": {Type: "character varying", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, + "table_name": {Type: "text", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "channel_no": {Type: "integer", IsNullable: "NO", Default: sql.NullString{Valid: false}, IsPrimary: true}, "total_events": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, "num_inserts": {Type: "bigint", IsNullable: "YES", Default: sql.NullString{Valid: false}, IsPrimary: false}, From 26fc27bfdd71bf315bf677596b0d310143e3c8ac Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 6 Jan 2025 11:08:16 +0530 Subject: [PATCH 087/105] separate callhome UnsupportedFeature for each extension in assess-migration (#2131) Add extension name to the FeatureName in callhome phase payload of assess-migration to have better categorization. --- yb-voyager/cmd/assessMigrationCommand.go | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index c54e998f56..754a1f8545 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -115,9 +115,7 @@ var assessMigrationCmd = &cobra.Command{ } // Assessment feature names to send the object names for to callhome -var featuresToSendObjectsToCallhome = []string{ - EXTENSION_FEATURE, -} +var featuresToSendObjectsToCallhome = []string{} func packAndSendAssessMigrationPayload(status string, errMsg string) { if !shouldSendCallhome() { @@ -163,24 +161,38 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { return len(constructs) }) - assessPayload := callhome.AssessMigrationPhasePayload{ - TargetDBVersion: assessmentReport.TargetDBVersion, - MigrationComplexity: assessmentReport.MigrationComplexity, - UnsupportedFeatures: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedFeatures, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature { + var unsupportedFeatures []callhome.UnsupportedFeature + for _, feature := range assessmentReport.UnsupportedFeatures { + if feature.FeatureName == EXTENSION_FEATURE { + // For extensions, we need to send the extension name in the feature name + // for better categorization in callhome + for _, object := range feature.Objects { + unsupportedFeatures = append(unsupportedFeatures, callhome.UnsupportedFeature{ + FeatureName: fmt.Sprintf("%s - %s", feature.FeatureName, object.ObjectName), + ObjectCount: 1, + TotalOccurrences: 1, + }) + } + } else { var objects []string if slices.Contains(featuresToSendObjectsToCallhome, feature.FeatureName) { objects = lo.Map(feature.Objects, func(o ObjectInfo, _ int) string { return o.ObjectName }) } - res := callhome.UnsupportedFeature{ + unsupportedFeatures = append(unsupportedFeatures, callhome.UnsupportedFeature{ FeatureName: feature.FeatureName, ObjectCount: len(feature.Objects), Objects: objects, TotalOccurrences: len(feature.Objects), - } - return res - })), + }) + } + } + + assessPayload := callhome.AssessMigrationPhasePayload{ + TargetDBVersion: assessmentReport.TargetDBVersion, + MigrationComplexity: assessmentReport.MigrationComplexity, + UnsupportedFeatures: callhome.MarshalledJsonString(unsupportedFeatures), UnsupportedQueryConstructs: callhome.MarshalledJsonString(countByConstructType), UnsupportedDatatypes: callhome.MarshalledJsonString(unsupportedDatatypesList), MigrationCaveats: callhome.MarshalledJsonString(lo.Map(assessmentReport.MigrationCaveats, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature { From 953591ce7527ab62a1b540c62971a906b9796440 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 13:37:23 +0530 Subject: [PATCH 088/105] Refactor migration assessment codepath: multiple minor changes (#2137) * Renamed AssessmentIssuePayload struct used for ybd to AssessmentIssueYugabyteD * Rename struct: Issue to AnalyzeSchemaIssue * Introduced AssessmentIssue struct for flattening the issues reported in AssessmentReport struct * fixing the struct in the unit test for checking non breaking upgrade --- yb-voyager/cmd/analyzeSchema.go | 60 ++++++-------- yb-voyager/cmd/assessMigrationCommand.go | 73 ++++++++--------- yb-voyager/cmd/common.go | 50 ++++++++---- yb-voyager/cmd/common_test.go | 78 ++++++++++--------- yb-voyager/cmd/constants.go | 6 +- yb-voyager/src/query/queryissue/constants.go | 8 +- .../src/query/queryissue/detectors_ddl.go | 4 +- yb-voyager/src/query/queryissue/issues_ddl.go | 24 +++--- .../src/query/queryissue/issues_ddl_test.go | 6 +- .../queryissue/parser_issue_detector_test.go | 2 +- yb-voyager/src/utils/commonVariables.go | 9 ++- 11 files changed, 163 insertions(+), 157 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index a4dda2b03e..d0dc86e9ab 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -194,43 +194,34 @@ var ( ) const ( - CONVERSION_ISSUE_REASON = "CREATE CONVERSION is not supported yet" - GIN_INDEX_MULTI_COLUMN_ISSUE_REASON = "Schema contains gin index on multi column which is not supported." - ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON = "Adding primary key to a partitioned table is not supported yet." - INHERITANCE_ISSUE_REASON = "TABLE INHERITANCE not supported in YugabyteDB" - CONSTRAINT_TRIGGER_ISSUE_REASON = "CONSTRAINT TRIGGER not supported yet." - REFERENCING_CLAUSE_FOR_TRIGGERS = "REFERENCING clause (transition tables) not supported yet." - BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE = "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers." - COMPOUND_TRIGGER_ISSUE_REASON = "COMPOUND TRIGGER not supported in YugabyteDB." - - STORED_GENERATED_COLUMN_ISSUE_REASON = "Stored generated columns are not supported." - UNSUPPORTED_EXTENSION_ISSUE = "This extension is not supported in YugabyteDB by default." - EXCLUSION_CONSTRAINT_ISSUE = "Exclusion constraint is not supported yet" - ALTER_TABLE_DISABLE_RULE_ISSUE = "ALTER TABLE name DISABLE RULE not supported yet" - STORAGE_PARAMETERS_DDL_STMT_ISSUE = "Storage parameters are not supported yet." - ALTER_TABLE_SET_ATTRIBUTE_ISSUE = "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet" - FOREIGN_TABLE_ISSUE_REASON = "Foreign tables require manual intervention." - ALTER_TABLE_CLUSTER_ON_ISSUE = "ALTER TABLE CLUSTER not supported yet." - DEFERRABLE_CONSTRAINT_ISSUE = "DEFERRABLE constraints not supported yet" - POLICY_ROLE_ISSUE = "Policy require roles to be created." - VIEW_CHECK_OPTION_ISSUE = "Schema containing VIEW WITH CHECK OPTION is not supported yet." - ISSUE_INDEX_WITH_COMPLEX_DATATYPES = `INDEX on column '%s' not yet supported` - ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES = `Primary key and Unique constraint on column '%s' not yet supported` - ISSUE_UNLOGGED_TABLE = "UNLOGGED tables are not supported yet." + // Issues detected using regexp, reported in assessment and analyze both + CONVERSION_ISSUE_REASON = "CREATE CONVERSION is not supported yet" + UNSUPPORTED_EXTENSION_ISSUE = "This extension is not supported in YugabyteDB by default." + VIEW_CHECK_OPTION_ISSUE = "Schema containing VIEW WITH CHECK OPTION is not supported yet." + + // Refactor: constants below used in some comparisions (use Issue Type there and remove these) + INHERITANCE_ISSUE_REASON = "TABLE INHERITANCE not supported in YugabyteDB" + ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON = "Adding primary key to a partitioned table is not supported yet." + COMPOUND_TRIGGER_ISSUE_REASON = "COMPOUND TRIGGER not supported in YugabyteDB." + STORED_GENERATED_COLUMN_ISSUE_REASON = "Stored generated columns are not supported." + FOREIGN_TABLE_ISSUE_REASON = "Foreign tables require manual intervention." + DEFERRABLE_CONSTRAINT_ISSUE = "DEFERRABLE constraints not supported yet" + POLICY_ROLE_ISSUE = "Policy require roles to be created." + ISSUE_INDEX_WITH_COMPLEX_DATATYPES = `INDEX on column '%s' not yet supported` + INDEX_METHOD_ISSUE_REASON = "Schema contains %s index which is not supported." + UNSUPPORTED_DATATYPE = "Unsupported datatype" UNSUPPORTED_DATATYPE_LIVE_MIGRATION = "Unsupported datatype for Live migration" UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB = "Unsupported datatype for Live migration with fall-forward/fallback" UNSUPPORTED_PG_SYNTAX = "Unsupported PG syntax" - INDEX_METHOD_ISSUE_REASON = "Schema contains %s index which is not supported." INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION = "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE" GIN_INDEX_DETAILS = "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." - SECURITY_INVOKER_VIEWS_ISSUE = "Security Invoker Views not supported yet" ) // Reports one case in JSON func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, issueType string, docsLink string) { - var issue utils.Issue + var issue utils.AnalyzeSchemaIssue issue.FilePath = filePath issue.Reason = reason issue.GH = ghIssue @@ -354,13 +345,6 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { // Checks compatibility of views func checkViews(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { - /*if dropMatViewRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "DROP MATERIALIZED VIEW not supported yet.",a - "https://github.com/YugaByte/yugabyte-db/issues/10102", "") - } else if view := matViewRegex.FindStringSubmatch(sqlInfo.stmt); view != nil { - reportCase(fpath, "Schema contains materialized view which is not supported. The view is: "+view[1], - "https://github.com/yugabyte/yugabyte-db/issues/10102", "") - } else */ if view := viewWithCheckRegex.FindStringSubmatch(sqlInfo.stmt); view != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, VIEW_CHECK_OPTION_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/22716", @@ -635,7 +619,7 @@ var MigrationCaveatsIssues = []string{ UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB, } -func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.Issue { +func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.AnalyzeSchemaIssue { issueType := UNSUPPORTED_FEATURES switch true { case isPlPgSQLIssue: @@ -685,7 +669,8 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true - return utils.Issue{ + return utils.AnalyzeSchemaIssue{ + IssueType: issueType, ObjectType: issueInstance.ObjectType, ObjectName: displayObjectName, Reason: issueInstance.TypeName, @@ -693,7 +678,6 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, - IssueType: issueType, Suggestion: issueInstance.Suggestion, GH: issueInstance.GH, MinimumVersionsFixedIn: issueInstance.MinimumVersionsFixedIn, @@ -1090,7 +1074,7 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source, detectIssues bool) utils. // Ideally all filtering of issues should happen in queryissue pkg layer, // but until we move all issue detection logic to queryissue pkg, we will filter issues here as well. - schemaAnalysisReport.Issues = lo.Filter(schemaAnalysisReport.Issues, func(i utils.Issue, index int) bool { + schemaAnalysisReport.Issues = lo.Filter(schemaAnalysisReport.Issues, func(i utils.AnalyzeSchemaIssue, index int) bool { fixed, err := i.IsFixedIn(targetDbVersion) if err != nil { utils.ErrExit("checking if issue %v is supported: %v", i, err) @@ -1254,7 +1238,7 @@ func packAndSendAnalyzeSchemaPayload(status string) { payload := createCallhomePayload() payload.MigrationPhase = ANALYZE_PHASE - var callhomeIssues []utils.Issue + var callhomeIssues []utils.AnalyzeSchemaIssue for _, issue := range schemaAnalysisReport.Issues { issue.SqlStatement = "" // Obfuscate sensitive information before sending to callhome cluster if !lo.ContainsBy(reasonsToSendObjectNameToCallhome, func(r string) bool { diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 754a1f8545..1c70f78b89 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -526,8 +526,8 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE } // flatten UnsupportedDataTypes, UnsupportedFeatures, MigrationCaveats -func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []AssessmentIssuePayload { - var issues []AssessmentIssuePayload +func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []AssessmentIssueYugabyteD { + var issues []AssessmentIssueYugabyteD var dataTypesDocsLink string switch source.DBType { @@ -537,7 +537,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK_ORACLE } for _, unsupportedDataType := range ar.UnsupportedDataTypes { - issues = append(issues, AssessmentIssuePayload{ + issues = append(issues, AssessmentIssueYugabyteD{ Type: DATATYPE, TypeDescription: DATATYPE_ISSUE_TYPE_DESCRIPTION, Subtype: unsupportedDataType.DataType, @@ -549,7 +549,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, unsupportedFeature := range ar.UnsupportedFeatures { for _, object := range unsupportedFeature.Objects { - issues = append(issues, AssessmentIssuePayload{ + issues = append(issues, AssessmentIssueYugabyteD{ Type: FEATURE, TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, Subtype: unsupportedFeature.FeatureName, @@ -564,7 +564,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, migrationCaveat := range ar.MigrationCaveats { for _, object := range migrationCaveat.Objects { - issues = append(issues, AssessmentIssuePayload{ + issues = append(issues, AssessmentIssueYugabyteD{ Type: MIGRATION_CAVEATS, TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, Subtype: migrationCaveat.FeatureName, @@ -578,7 +578,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment } for _, uqc := range ar.UnsupportedQueryConstructs { - issues = append(issues, AssessmentIssuePayload{ + issues = append(issues, AssessmentIssueYugabyteD{ Type: QUERY_CONSTRUCT, TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, Subtype: uqc.ConstructTypeName, @@ -590,7 +590,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { for _, object := range plpgsqlObjects.Objects { - issues = append(issues, AssessmentIssuePayload{ + issues = append(issues, AssessmentIssueYugabyteD{ Type: PLPGSQL_OBJECT, TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, Subtype: plpgsqlObjects.FeatureName, @@ -914,6 +914,12 @@ func generateAssessmentReport() (err error) { } func getAssessmentReportContentFromAnalyzeSchema() error { + /* + Here we are generating analyze schema report which converts issue instance to analyze schema issue + Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) + + But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). + */ schemaAnalysisReport := analyzeSchemaInternal(&source, true) assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary @@ -1009,31 +1015,28 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) } - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, STORED_GENERATED_COLUMN_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, "", queryissue.STORED_GENERATED_COLUMNS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, GIN_INDEX_MULTI_COLUMN_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, ALTER_TABLE_SET_ATTRIBUTE_ISSUE, "", schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, ALTER_TABLE_DISABLE_RULE_ISSUE, "", schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, ALTER_TABLE_CLUSTER_ON_ISSUE, "", schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, STORAGE_PARAMETERS_DDL_STMT_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, "", queryissue.MULTI_COLUMN_GIN_INDEX, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, "", queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, "", queryissue.ALTER_TABLE_DISABLE_RULE, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, "", queryissue.ALTER_TABLE_CLUSTER_ON, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, "", queryissue.STORAGE_PARAMETER, schemaAnalysisReport, true, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, EXCLUSION_CONSTRAINT_ISSUE, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, "", queryissue.EXCLUSION_CONSTRAINTS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, "", queryissue.DEFERRABLE_CONSTRAINTS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)) - - pkOrUkConstraintIssuePrefix := strings.Split(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, "%s")[0] - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, "", schemaAnalysisReport, false, "")) - - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, queryissue.ADVISORY_LOCKS_NAME, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, queryissue.XML_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, "", queryissue.PK_UK_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, "", queryissue.UNLOGGED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, "", queryissue.REFERENCING_CLAUSE_IN_TRIGGER, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, "", queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, "", queryissue.ADVISORY_LOCKS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, "", queryissue.XML_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, "", queryissue.SYSTEM_COLUMNS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", queryissue.LARGE_OBJECT_FUNCTIONS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) @@ -1046,7 +1049,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem }), nil } -func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.SchemaReport, unsupportedIndexDatatypes []string) UnsupportedFeature { +func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.SchemaReport, unsupportedIndexDatatypes []string) UnsupportedFeature { // TODO: include MinimumVersionsFixedIn indexesOnComplexTypesFeature := UnsupportedFeature{ FeatureName: "Index on complex datatypes", @@ -1134,10 +1137,10 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U return nil } analyzeIssues := schemaAnalysisReport.Issues - plpgsqlIssues := lo.Filter(analyzeIssues, func(issue utils.Issue, _ int) bool { + plpgsqlIssues := lo.Filter(analyzeIssues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { return issue.IssueType == UNSUPPORTED_PLPGSQL_OBEJCTS }) - groupPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.Issue) string { + groupPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.AnalyzeSchemaIssue) string { return issue.Reason }) var unsupportedPlpgSqlObjects []UnsupportedFeature @@ -1170,8 +1173,8 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U } unsupportedPlpgSqlObjects = append(unsupportedPlpgSqlObjects, feature) } - return unsupportedPlpgSqlObjects + return unsupportedPlpgSqlObjects } func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error) { @@ -1416,11 +1419,11 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration case POSTGRESQL: log.Infof("add migration caveats to assessment report") migrationCaveats := make([]UnsupportedFeature, 0) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, "", + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, "", queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, schemaAnalysisReport, true, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, FOREIGN_TABLE_ISSUE_REASON, "", + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, POLICY_ROLE_ISSUE, "", + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_ISSUE)) if len(unsupportedDataTypesForLiveMigration) > 0 { diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 468debe2da..47c5a7c496 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1176,20 +1176,36 @@ func getMigrationComplexityForOracle(schemaDirectory string) (string, error) { // TODO: consider merging all unsupported field with single AssessmentReport struct member as AssessmentIssue type AssessmentReport struct { - VoyagerVersion string `json:"VoyagerVersion"` - TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` - MigrationComplexity string `json:"MigrationComplexity"` - SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` - Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` - UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` - UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` - UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` - UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` - UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` - UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` - MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` - TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` - Notes []string `json:"Notes"` + VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` + MigrationComplexity string `json:"MigrationComplexity"` + SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` + Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` + Issues []AssessmentIssue `json:"-"` // disabled in reports till corresponding UI changes are done(json and html reports) + TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` + Notes []string `json:"Notes"` + + // fields going to be deprecated + UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` + UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` + UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` + UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` + UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` + UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` + MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` +} + +type AssessmentIssue struct { + Category string + CategoryDescription string + TypeName string + TypeDescription string + Impact string + ObjectType string + ObjectName string + SqlStatement string + DocsLink string + MinimumVersionFixedIn map[string]*ybversion.YBVersion } type UnsupportedFeature struct { @@ -1240,15 +1256,15 @@ type AssessMigrationPayload struct { TargetDBVersion *ybversion.YBVersion MigrationComplexity string SchemaSummary utils.SchemaSummary - AssessmentIssues []AssessmentIssuePayload + AssessmentIssues []AssessmentIssueYugabyteD SourceSizeDetails SourceDBSizeDetails TargetRecommendations TargetSizingRecommendations - ConversionIssues []utils.Issue + ConversionIssues []utils.AnalyzeSchemaIssue // Depreacted: AssessmentJsonReport is depricated; use the fields directly inside struct AssessmentJsonReport AssessmentReport } -type AssessmentIssuePayload struct { +type AssessmentIssueYugabyteD struct { Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index 49eee13b08..6dd45e6669 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -134,6 +134,9 @@ func TestAssessmentReportStructs(t *testing.T) { MigrationComplexity string `json:"MigrationComplexity"` SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` + Issues []AssessmentIssue `json:"-"` + TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` + Notes []string `json:"Notes"` UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` @@ -141,8 +144,6 @@ func TestAssessmentReportStructs(t *testing.T) { UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` - TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` - Notes []string `json:"Notes"` }{}, }, } @@ -196,6 +197,24 @@ func TestAssessmentReportJson(t *testing.T) { }, FailureReasoning: "Test failure reasoning", }, + Issues: nil, + TableIndexStats: &[]migassessment.TableIndexStats{ + { + SchemaName: "public", + ObjectName: "test_table", + RowCount: Int64Ptr(100), + ColumnCount: Int64Ptr(10), + Reads: Int64Ptr(100), + Writes: Int64Ptr(100), + ReadsPerSecond: Int64Ptr(10), + WritesPerSecond: Int64Ptr(10), + IsIndex: true, + ObjectType: "Table", + ParentTableName: StringPtr("parent_table"), + SizeInBytes: Int64Ptr(1024), + }, + }, + Notes: []string{"Test note"}, UnsupportedDataTypes: []utils.TableColumnsDataTypes{ { SchemaName: "public", @@ -262,23 +281,6 @@ func TestAssessmentReportJson(t *testing.T) { MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{"2024.1.1": newYbVersion}, }, }, - TableIndexStats: &[]migassessment.TableIndexStats{ - { - SchemaName: "public", - ObjectName: "test_table", - RowCount: Int64Ptr(100), - ColumnCount: Int64Ptr(10), - Reads: Int64Ptr(100), - Writes: Int64Ptr(100), - ReadsPerSecond: Int64Ptr(10), - WritesPerSecond: Int64Ptr(10), - IsIndex: true, - ObjectType: "Table", - ParentTableName: StringPtr("parent_table"), - SizeInBytes: Int64Ptr(1024), - }, - }, - Notes: []string{"Test note"}, } // Make the report directory @@ -341,6 +343,25 @@ func TestAssessmentReportJson(t *testing.T) { }, "FailureReasoning": "Test failure reasoning" }, + "TableIndexStats": [ + { + "SchemaName": "public", + "ObjectName": "test_table", + "RowCount": 100, + "ColumnCount": 10, + "Reads": 100, + "Writes": 100, + "ReadsPerSecond": 10, + "WritesPerSecond": 10, + "IsIndex": true, + "ObjectType": "Table", + "ParentTableName": "parent_table", + "SizeInBytes": 1024 + } + ], + "Notes": [ + "Test note" + ], "UnsupportedDataTypes": [ { "SchemaName": "public", @@ -411,25 +432,6 @@ func TestAssessmentReportJson(t *testing.T) { "2024.1.1": "2024.1.1.1" } } - ], - "TableIndexStats": [ - { - "SchemaName": "public", - "ObjectName": "test_table", - "RowCount": 100, - "ColumnCount": 10, - "Reads": 100, - "Writes": 100, - "ReadsPerSecond": 10, - "WritesPerSecond": 10, - "IsIndex": true, - "ObjectType": "Table", - "ParentTableName": "parent_table", - "SizeInBytes": 1024 - } - ], - "Notes": [ - "Test note" ] }` diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index a56ec10970..f4d9cc4fcf 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -173,7 +173,7 @@ const ( List of all the features we are reporting as part of Unsupported features and Migration caveats */ const ( - // Types for AssessmentIssue + // AssessmentIssue types used in YugabyteD payload FEATURE = "feature" DATATYPE = "datatype" QUERY_CONSTRUCT = "query_construct" // confused: in json for some values we are using space separated and for some snake_case @@ -227,11 +227,11 @@ const ( POLICIES_CAVEAT_FEATURE = "Policies" UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration" UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration with Fall-forward/Fallback" + UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE = "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." + UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE = "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." DESCRIPTION_ADD_PK_TO_PARTITION_TABLE = `After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.` DESCRIPTION_FOREIGN_TABLES = `During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.` DESCRIPTION_POLICY_ROLE_ISSUE = `There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.` - UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE = "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." - UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE = "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." ) var supportedSourceDBTypes = []string{ORACLE, MYSQL, POSTGRESQL, YUGABYTEDB} diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 586957bf92..117e6dbdf2 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -23,16 +23,16 @@ const ( UNLOGGED_TABLE = "UNLOGGED_TABLE" UNSUPPORTED_INDEX_METHOD = "UNSUPPORTED_INDEX_METHOD" STORAGE_PARAMETER = "STORAGE_PARAMETER" - SET_ATTRIBUTES = "SET_ATTRIBUTES" - CLUSTER_ON = "CLUSTER_ON" - DISABLE_RULE = "DISABLE_RULE" + ALTER_TABLE_SET_COLUMN_ATTRIBUTE = "ALTER_TABLE_SET_COLUMN_ATTRIBUTE" + ALTER_TABLE_CLUSTER_ON = "ALTER_TABLE_CLUSTER_ON" + ALTER_TABLE_DISABLE_RULE = "ALTER_TABLE_DISABLE_RULE" EXCLUSION_CONSTRAINTS = "EXCLUSION_CONSTRAINTS" DEFERRABLE_CONSTRAINTS = "DEFERRABLE_CONSTRAINTS" MULTI_COLUMN_GIN_INDEX = "MULTI_COLUMN_GIN_INDEX" ORDERED_GIN_INDEX = "ORDERED_GIN_INDEX" POLICY_WITH_ROLES = "POLICY_WITH_ROLES" CONSTRAINT_TRIGGER = "CONSTRAINT_TRIGGER" - REFERENCING_CLAUSE_FOR_TRIGGERS = "REFERENCING_CLAUSE_FOR_TRIGGERS" + REFERENCING_CLAUSE_IN_TRIGGER = "REFERENCING_CLAUSE_IN_TRIGGER" BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE = "BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE" ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE = "ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE" EXPRESSION_PARTITION_WITH_PK_UK = "EXPRESSION_PARTITION_WITH_PK_UK" diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 595680f9ae..c63a1e9799 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -413,7 +413,7 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q switch alter.AlterType { case queryparser.SET_OPTIONS: if alter.NumSetAttributes > 0 { - issues = append(issues, NewSetAttributeIssue( + issues = append(issues, NewSetColumnAttributeIssue( obj.GetObjectType(), alter.GetObjectName(), "", // query string @@ -475,7 +475,7 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q } case queryparser.DISABLE_RULE: - issues = append(issues, NewDisableRuleIssue( + issues = append(issues, NewAlterTableDisableRuleIssue( obj.GetObjectType(), alter.GetObjectName(), "", // query string diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index ba1d4cdaed..d005a1ef3b 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -80,21 +80,21 @@ func NewStorageParameterIssue(objectType string, objectName string, sqlStatement return newQueryIssue(storageParameterIssue, objectType, objectName, sqlStatement, details) } -var setAttributeIssue = issue.Issue{ - Type: SET_ATTRIBUTES, +var setColumnAttributeIssue = issue.Issue{ + Type: ALTER_TABLE_SET_COLUMN_ATTRIBUTE, TypeName: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", } -func NewSetAttributeIssue(objectType string, objectName string, sqlStatement string) QueryIssue { +func NewSetColumnAttributeIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - return newQueryIssue(setAttributeIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(setColumnAttributeIssue, objectType, objectName, sqlStatement, details) } -var clusterOnIssue = issue.Issue{ - Type: CLUSTER_ON, +var alterTableClusterOnIssue = issue.Issue{ + Type: ALTER_TABLE_CLUSTER_ON, TypeName: "ALTER TABLE CLUSTER not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema.", @@ -103,20 +103,20 @@ var clusterOnIssue = issue.Issue{ func NewClusterONIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - return newQueryIssue(clusterOnIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(alterTableClusterOnIssue, objectType, objectName, sqlStatement, details) } -var disableRuleIssue = issue.Issue{ - Type: DISABLE_RULE, +var alterTableDisableRuleIssue = issue.Issue{ + Type: ALTER_TABLE_DISABLE_RULE, TypeName: "ALTER TABLE name DISABLE RULE not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove this and the rule '%s' from the exported schema to be not enabled on the table.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", } -func NewDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) QueryIssue { +func NewAlterTableDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) QueryIssue { details := map[string]interface{}{} - issue := disableRuleIssue + issue := alterTableDisableRuleIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, ruleName) return newQueryIssue(issue, objectType, objectName, sqlStatement, details) } @@ -200,7 +200,7 @@ func NewConstraintTriggerIssue(objectType string, objectName string, sqlStatemen } var referencingClauseInTriggerIssue = issue.Issue{ - Type: REFERENCING_CLAUSE_FOR_TRIGGERS, + Type: REFERENCING_CLAUSE_IN_TRIGGER, TypeName: "REFERENCING clause (transition tables) not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1668", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index d53896781f..709dd369d0 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -141,7 +141,7 @@ func testSetAttributeIssue(t *testing.T) { ); ALTER TABLE ONLY public.event_search ALTER COLUMN room_id SET (n_distinct=-0.01)`) - assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE ALTER column not supported yet", setAttributeIssue) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE ALTER column not supported yet", setColumnAttributeIssue) } func testClusterOnIssue(t *testing.T) { @@ -161,7 +161,7 @@ func testClusterOnIssue(t *testing.T) { ALTER TABLE public.test CLUSTER ON test_age_salary`) - assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE CLUSTER not supported yet", clusterOnIssue) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE CLUSTER not supported yet", alterTableClusterOnIssue) } func testDisableRuleIssue(t *testing.T) { @@ -177,7 +177,7 @@ func testDisableRuleIssue(t *testing.T) { ALTER TABLE trule DISABLE RULE trule_rule`) - assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE DISABLE RULE not supported yet", disableRuleIssue) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ALTER TABLE DISABLE RULE not supported yet", alterTableDisableRuleIssue) } func testStorageParameterIssue(t *testing.T) { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e5b6315b0c..c09c4e59d6 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -197,7 +197,7 @@ func TestAllIssues(t *testing.T) { NewStorageParameterIssue("INDEX", "abc ON public.example", stmt3), }, stmt4: []QueryIssue{ - NewDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), + NewAlterTableDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), }, stmt5: []QueryIssue{ NewDeferrableConstraintIssue("TABLE", "abc", stmt5, "cnstr_id"), diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index c1d51326bc..b42c547d98 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -78,7 +78,7 @@ type SchemaReport struct { TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` MigrationComplexity string `json:"MigrationComplexity"` SchemaSummary SchemaSummary `json:"Summary"` - Issues []Issue `json:"Issues"` + Issues []AnalyzeSchemaIssue `json:"Issues"` } type SchemaSummary struct { @@ -90,7 +90,7 @@ type SchemaSummary struct { DBObjects []DBObject `json:"DatabaseObjects"` } -//TODO: Rename the variables of TotalCount and InvalidCount -> TotalObjects and ObjectsWithIssues +// TODO: Rename the variables of TotalCount and InvalidCount -> TotalObjects and ObjectsWithIssues type DBObject struct { ObjectType string `json:"ObjectType"` TotalCount int `json:"TotalCount"` @@ -100,7 +100,8 @@ type DBObject struct { } // TODO: support MinimumVersionsFixedIn in xml -type Issue struct { +type AnalyzeSchemaIssue struct { + // TODO: rename IssueType to Category IssueType string `json:"IssueType"` //category: unsupported_features, unsupported_plpgsql_objects, etc ObjectType string `json:"ObjectType"` ObjectName string `json:"ObjectName"` @@ -114,7 +115,7 @@ type Issue struct { MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn" xml:"-"` // key: series (2024.1, 2.21, etc) } -func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { +func (i AnalyzeSchemaIssue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { if i.MinimumVersionsFixedIn == nil { return false, nil } From 16b08feea3862f71be63c1a9e3ba30f1cbeb5f4f Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Mon, 6 Jan 2025 16:49:40 +0530 Subject: [PATCH 089/105] Reporting range aggregate functions and IS JSON predicate clauses (#2113) Reporting the IS JSON predicate clauses - https://yugabyte.atlassian.net/browse/DB-14543 Reporting the range_agg, range_intersect_agg - range type functions - https://yugabyte.atlassian.net/browse/DB-14224 --- .../dummy-export-dir/schema/tables/table.sql | 17 +++- .../tests/analyze-schema/expected_issues.json | 62 +++++++++++-- migtests/tests/analyze-schema/summary.json | 6 +- .../expectedAssessmentReport.json | 91 ++++++++++++++++-- .../pg_assessment_report_uqc.sql | 42 ++++++++- .../unsupported_query_constructs.sql | 14 ++- yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/src/query/queryissue/constants.go | 2 + yb-voyager/src/query/queryissue/detectors.go | 32 +++++++ .../src/query/queryissue/detectors_test.go | 22 ++--- yb-voyager/src/query/queryissue/helpers.go | 2 +- yb-voyager/src/query/queryissue/issues_dml.go | 19 +++- .../src/query/queryissue/issues_dml_test.go | 66 +++++++++++++ .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 93 ++++++++++++++++++- .../src/query/queryparser/traversal_proto.go | 1 + 16 files changed, 428 insertions(+), 43 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 4da2e5f444..5464246d5c 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -362,30 +362,39 @@ CREATE TABLE test_udt ( employee_name VARCHAR(100), home_address address_type, some_field enum_test, - home_address1 non_public.address_type1 + home_address1 non_public.address_type1, + scalar_column TEXT CHECK (scalar_column IS JSON SCALAR) ); CREATE TABLE test_arr_enum ( id int, arr text[], - arr_enum enum_test[] + arr_enum enum_test[], + object_column TEXT CHECK (object_column IS JSON OBJECT) ); CREATE TABLE public.locations ( id integer NOT NULL, name character varying(100), - geom geometry(Point,4326) + geom geometry(Point,4326), + array_column TEXT CHECK (array_column IS JSON ARRAY) ); CREATE TABLE public.xml_data_example ( id SERIAL PRIMARY KEY, name VARCHAR(255), description XML DEFAULT xmlparse(document 'Default Product100.00Electronics'), - created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) ); CREATE TABLE image (title text, raster lo); +-- IS JSON Predicate +CREATE TABLE public.json_data ( + id SERIAL PRIMARY KEY, + data_column TEXT NOT NULL CHECK (data_column IS JSON) +); CREATE TABLE employees (id INT PRIMARY KEY, salary INT); -- create table with multirange data types diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 0bfd66b0e5..0fc61bbf66 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -30,6 +30,56 @@ "GH": "", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.json_data", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE public.json_data (\n id SERIAL PRIMARY KEY,\n data_column TEXT NOT NULL CHECK (data_column IS JSON)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "test_arr_enum", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n object_column TEXT CHECK (object_column IS JSON OBJECT)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "test_udt", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.xml_data_example", + "Reason": "Json Type Predicate", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.locations", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326),\n array_column TEXT CHECK (array_column IS JSON ARRAY)\n );", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "VIEW", @@ -55,7 +105,7 @@ "ObjectType": "TABLE", "ObjectName": "public.xml_data_example", "Reason": "XML Functions", - "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", @@ -66,7 +116,7 @@ "ObjectType": "TABLE", "ObjectName": "public.xml_data_example", "Reason": "Unsupported datatype - xml on column - description", - "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", @@ -275,7 +325,7 @@ "ObjectType": "TABLE", "ObjectName": "test_udt", "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - address_type on column - home_address", - "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);", + "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", @@ -286,7 +336,7 @@ "ObjectType": "TABLE", "ObjectName": "test_arr_enum", "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - enum_test[] on column - arr_enum", - "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[]\n);", + "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n object_column TEXT CHECK (object_column IS JSON OBJECT)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", @@ -297,7 +347,7 @@ "ObjectType": "TABLE", "ObjectName": "test_udt", "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - non_public.address_type1 on column - home_address1", - "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);", + "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", @@ -1212,7 +1262,7 @@ "ObjectType": "TABLE", "ObjectName": "public.locations", "Reason": "Unsupported datatype - geometry on column - geom", - "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326)\n );", + "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326),\n array_column TEXT CHECK (array_column IS JSON ARRAY)\n );", "Suggestion": "", "GH": "https://github.com/yugabyte/yugabyte-db/issues/11323", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 5e9353c7ac..8cc96548fa 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 58, - "InvalidCount": 49, - "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + "TotalCount": 59, + "InvalidCount": 50, + "ObjectNames": "public.json_data, employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, { "ObjectType": "INDEX", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index d02458c005..276d6f01e5 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -25,15 +25,15 @@ }, { "ObjectType": "TABLE", - "TotalCount": 2, - "InvalidCount": 0, - "ObjectNames": "analytics.metrics, sales.orders" + "TotalCount": 4, + "InvalidCount": 1, + "ObjectNames": "sales.json_data, analytics.metrics, sales.events, sales.orders" }, { "ObjectType": "VIEW", - "TotalCount": 1, - "InvalidCount": 1, - "ObjectNames": "sales.employ_depart_view" + "TotalCount": 3, + "InvalidCount": 3, + "ObjectNames": "sales.event_analysis_view, sales.event_analysis_view2, sales.employ_depart_view" } ] }, @@ -41,9 +41,11 @@ "SizingRecommendation": { "ColocatedTables": [ "sales.orders", - "analytics.metrics" + "analytics.metrics", + "sales.events", + "sales.json_data" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 2 objects (2 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 4 objects (4 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": null, "NumNodes": 3, "VCPUsPerInstance": 4, @@ -55,18 +57,43 @@ }, "FailureReasoning": "" }, - "UnsupportedDataTypes": null, + "UnsupportedDataTypes": [ + { + "SchemaName": "sales", + "TableName": "event_analysis_view", + "ColumnName": "all_event_ranges", + "DataType": "datemultirange" + } + ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ { "FeatureName": "Aggregate Functions", "Objects": [ + { + "ObjectName": "sales.event_analysis_view", + "SqlStatement": "CREATE VIEW sales.event_analysis_view AS\n SELECT range_agg(event_range) AS all_event_ranges\n FROM sales.events;" + }, + { + "ObjectName": "sales.event_analysis_view2", + "SqlStatement": "CREATE VIEW sales.event_analysis_view2 AS\n SELECT range_intersect_agg(event_range) AS overlapping_range\n FROM sales.events;" + }, { "ObjectName": "sales.employ_depart_view", "SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n FROM public.employees;" } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Json Type Predicate", + "Objects": [ + { + "ObjectName": "sales.json_data", + "SqlStatement": "CREATE TABLE sales.json_data (\n id integer NOT NULL,\n array_column text,\n unique_keys_column text,\n CONSTRAINT json_data_array_column_check CHECK ((array_column IS JSON ARRAY)),\n CONSTRAINT json_data_unique_keys_column_check CHECK ((unique_keys_column IS JSON WITH UNIQUE KEYS))\n);" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -85,6 +112,34 @@ "ParentTableName": null, "SizeInBytes": 8192 }, + { + "SchemaName": "sales", + "ObjectName": "events", + "RowCount": 3, + "ColumnCount": 2, + "Reads": 6, + "Writes": 3, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, + { + "SchemaName": "sales", + "ObjectName": "json_data", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "analytics", "ObjectName": "metrics", @@ -109,11 +164,29 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, + { + "ConstructTypeName": "Aggregate Functions", + "Query": "SELECT range_intersect_agg(event_range) AS intersection_of_ranges\nFROM sales.events", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Aggregate Functions", + "Query": "SELECT range_agg(event_range) AS union_of_ranges\nFROM sales.events", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "Aggregate Functions", "Query": "SELECT\n any_value(name) AS any_employee\n FROM employees", "DocsLink": "", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Json Type Predicate", + "Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": null diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index 2b42d33348..b4fe358748 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -45,4 +45,44 @@ INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRat create view sales.employ_depart_view AS SELECT any_value(name) AS any_employee - FROM employees; \ No newline at end of file + FROM employees; + +CREATE TABLE sales.events ( + id int PRIMARY KEY, + event_range daterange +); + +-- Insert some ranges +INSERT INTO sales.events (id, event_range) VALUES + (1,'[2024-01-01, 2024-01-10]'::daterange), + (2,'[2024-01-05, 2024-01-15]'::daterange), + (3,'[2024-01-20, 2024-01-25]'::daterange); + +CREATE VIEW sales.event_analysis_view AS +SELECT + range_agg(event_range) AS all_event_ranges +FROM + sales.events; + +CREATE VIEW sales.event_analysis_view2 AS +SELECT + range_intersect_agg(event_range) AS overlapping_range +FROM + sales.events; + +-- PG 16 and above feature +CREATE TABLE sales.json_data ( + id int PRIMARY KEY, + array_column TEXT CHECK (array_column IS JSON ARRAY), + unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) +); + +INSERT INTO public.json_data ( + id, data_column, object_column, array_column, scalar_column, unique_keys_column +) VALUES ( + 1, '{"key": "value"}', + 2, '{"name": "John", "age": 30}', + 3, '[1, 2, 3, 4]', + 4, '"hello"', + 5, '{"uniqueKey1": "value1", "uniqueKey2": "value2"}' +); \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index 1840c36242..43a68b8e37 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -25,4 +25,16 @@ WHERE metric_value > 0.02; -- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15 SELECT any_value(name) AS any_employee - FROM employees; \ No newline at end of file + FROM employees; + +--PG15 +SELECT range_agg(event_range) AS union_of_ranges +FROM sales.events; + +SELECT range_intersect_agg(event_range) AS intersection_of_ranges +FROM sales.events; + +-- -- PG 16 and above feature +SELECT * +FROM sales.json_data +WHERE array_column IS JSON ARRAY; \ No newline at end of file diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 1c70f78b89..d007fa217f 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1043,6 +1043,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 117e6dbdf2..63c89270e0 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -51,6 +51,8 @@ const ( AGGREGATE_FUNCTION = "AGGREGATE_FUNCTION" AGGREGATION_FUNCTIONS_NAME = "Aggregate Functions" + JSON_TYPE_PREDICATE = "JSON_TYPE_PREDICATE" + JSON_TYPE_PREDICATE_NAME = "Json Type Predicate" JSON_CONSTRUCTOR_FUNCTION = "JSON_CONSTRUCTOR_FUNCTION" JSON_CONSTRUCTOR_FUNCTION_NAME = "Json Constructor Functions" JSON_QUERY_FUNCTION = "JSON_QUERY_FUNCTION" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index c834fa6db0..d1ec43c5e9 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -390,3 +390,35 @@ func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { } return issues } + +type JsonPredicateExprDetector struct { + query string + detected bool +} + +func NewJsonPredicateExprDetector(query string) *JsonPredicateExprDetector { + return &JsonPredicateExprDetector{ + query: query, + } +} + +func (j *JsonPredicateExprDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_IS_PREDICATE_NODE { + /* + SELECT js IS JSON "json?" FROM (VALUES ('123')) foo(js); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{string:{sval:"js"}} location:337}} location:337}} + target_list:{res_target:{name:"json?" val:{json_is_predicate:{expr:{column_ref:{fields:{string:{sval:"js"}} location:341}} + format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT location:-1} item_type:JS_TYPE_ANY location:341}} location:341}} ... + */ + j.detected = true + } + return nil +} + +func (j *JsonPredicateExprDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if j.detected { + issues = append(issues, NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", j.query)) + } + return issues +} diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 16c136d3bc..e1aa7a32aa 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -98,13 +98,6 @@ func TestFuncCallDetector(t *testing.T) { `SELECT pg_advisory_unlock_all();`, } - anyValAggSqls := []string{ - `SELECT - department, - any_value(employee_name) AS any_employee - FROM employees - GROUP BY department;`, - } loFunctionSqls := []string{ `UPDATE documents SET content_oid = lo_import('/path/to/new/file.pdf') @@ -130,13 +123,9 @@ WHERE title = 'Design Document';`, issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) assert.Equal(t, len(issues), 1) assert.Equal(t, issues[0].Type, LARGE_OBJECT_FUNCTIONS, "Large Objects not detected in SQL: %s", sql) - } - for _, sql := range anyValAggSqls { - issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) - assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) - assert.Equal(t, AGGREGATE_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } + } func TestColumnRefDetector(t *testing.T) { @@ -710,3 +699,12 @@ WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);` assert.Equal(t, JSON_QUERY_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } + +func TestIsJsonPredicate(t *testing.T) { + sql := `SELECT js, js IS JSON "json?" FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);` + + issues := getDetectorIssues(t, NewJsonPredicateExprDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSON_TYPE_PREDICATE, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + +} diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index d24a14ed53..12f2a2e245 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -104,7 +104,7 @@ var UnsupportedIndexDatatypes = []string{ var unsupportedAggFunctions = mapset.NewThreadUnsafeSet([]string{ //agg function added in PG16 - https://www.postgresql.org/docs/16/functions-aggregate.html#id-1.5.8.27.5.2.4.1.1.1.1 - "any_value", + "any_value", "range_agg", "range_intersect_agg", }...) const ( diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 7e6f8aed88..0752051de2 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -74,10 +74,10 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s return newQueryIssue(regexFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var anyValueAggFunctionIssue = issue.Issue{ +var aggregateFunctionIssue = issue.Issue{ Type: AGGREGATE_FUNCTION, TypeName: AGGREGATION_FUNCTIONS_NAME, - TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + TypeDescription: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", Suggestion: "", GH: "", DocsLink: "", @@ -88,7 +88,7 @@ func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatem details := map[string]interface{}{ FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports } - return newQueryIssue(anyValueAggFunctionIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(aggregateFunctionIssue, objectType, objectName, sqlStatement, details) } var jsonConstructorFunctionsIssue = issue.Issue{ @@ -142,6 +142,19 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } +var jsonPredicateIssue = issue.Issue{ + Type: JSON_TYPE_PREDICATE, + TypeName: JSON_TYPE_PREDICATE_NAME, + TypeDescription: "IS JSON predicate expressions not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", //TODO +} + +func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(jsonPredicateIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var copyFromWhereIssue = issue.Issue{ Type: COPY_FROM_WHERE, TypeName: "COPY FROM ... WHERE", diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index cff5910c26..6825eb1af0 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -122,6 +122,17 @@ func testJsonConstructorFunctions(t *testing.T) { } } +func testJsonPredicateIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, `SELECT js, js IS JSON "json?" FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "JSON"`, jsonConstructorFunctionsIssue) +} + func testJsonQueryFunctions(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -159,6 +170,55 @@ WHERE JSON_EXISTS(details, '$.author');`, assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "COLUMNS"`, jsonConstructorFunctionsIssue) } +func testAggFunctions(t *testing.T) { + sqls := []string{ + `CREATE TABLE any_value_ex ( + department TEXT, + employee_name TEXT, + salary NUMERIC +); + +INSERT INTO any_value_ex VALUES +('HR', 'Alice', 50000), +('HR', 'Bob', 55000), +('IT', 'Charlie', 60000), +('IT', 'Diana', 62000); + +SELECT + department, + any_value(employee_name) AS any_employee +FROM any_value_ex +GROUP BY department;`, + +`CREATE TABLE events ( + id SERIAL PRIMARY KEY, + event_range daterange +); + +INSERT INTO events (event_range) VALUES + ('[2024-01-01, 2024-01-10]'::daterange), + ('[2024-01-05, 2024-01-15]'::daterange), + ('[2024-01-20, 2024-01-25]'::daterange); + +SELECT range_agg(event_range) AS union_of_ranges +FROM events; + +SELECT range_intersect_agg(event_range) AS intersection_of_ranges +FROM events;`, + } + + for _, sql := range sqls { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `does not exist`, aggregateFunctionIssue) + } +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -204,4 +264,10 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "json query functions", ybVersion), testJsonQueryFunctions) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "aggregate functions", ybVersion), testAggFunctions) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "json type predicate", ybVersion), testJsonPredicateIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index c7faf12092..e4db26c0fd 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -379,6 +379,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), + NewJsonPredicateExprDetector(query), } processor := func(msg protoreflect.Message) error { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index c09c4e59d6..28c251c785 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -25,9 +25,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/samber/lo" "github.com/stretchr/testify/assert" - testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) const ( @@ -546,6 +546,19 @@ FROM books;`, `SELECT id, JSON_VALUE(details, '$.title') AS title FROM books WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`, + `SELECT js, js IS JSON "json?", js IS JSON SCALAR "scalar?", js IS JSON OBJECT "object?", js IS JSON ARRAY "array?" +FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);`, + `SELECT js, + js IS JSON OBJECT "object?", + js IS JSON ARRAY "array?", + js IS JSON ARRAY WITH UNIQUE KEYS "array w. UK?", + js IS JSON ARRAY WITHOUT UNIQUE KEYS "array w/o UK?" +FROM (VALUES ('[{"a":"1"}, + {"b":"2","b":"3"}]')) foo(js);`, + `SELECT js, + js IS JSON OBJECT "object?" + FROM (VALUES ('[{"a":"1"}, + {"b":"2","b":"3"}]')) foo(js); `, `CREATE MATERIALIZED VIEW public.test_jsonb_view AS SELECT id, @@ -560,6 +573,11 @@ JSON_TABLE(data, '$.skills[*]' ) ) AS jt;`, `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, +`CREATE TABLE sales.json_data ( + id int PRIMARY KEY, + array_column TEXT CHECK (array_column IS JSON ARRAY), + unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) +);`, } sqlsWithExpectedIssues := map[string][]QueryIssue{ sqls[0]: []QueryIssue{ @@ -604,10 +622,22 @@ JSON_TABLE(data, '$.skills[*]' NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[12], []string{JSON_VALUE, JSON_EXISTS}), }, sqls[13]: []QueryIssue{ - NewJsonQueryFunctionIssue("MVIEW", "public.test_jsonb_view", sqls[13], []string{JSON_VALUE, JSON_EXISTS, JSON_TABLE}), + NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", sqls[13]), }, sqls[14]: []QueryIssue{ - NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[14], []string{JSON_ARRAY}), + NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", sqls[14]), + }, + sqls[15]: []QueryIssue{ + NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", sqls[15]), + }, + sqls[16]: []QueryIssue{ + NewJsonQueryFunctionIssue("MVIEW", "public.test_jsonb_view", sqls[16], []string{JSON_VALUE, JSON_EXISTS, JSON_TABLE}), + }, + sqls[17]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[17], []string{JSON_ARRAY}), + }, + sqls[18]: []QueryIssue{ + NewJsonPredicateIssue("TABLE", "sales.json_data", sqls[18]), }, } parserIssueDetector := NewParserIssueDetector() @@ -623,6 +653,63 @@ JSON_TABLE(data, '$.skills[*]' } } } + +func TestAggregateFunctions(t *testing.T) { + sqls := []string{ + `SELECT + department, + any_value(employee_name) AS any_employee + FROM employees + GROUP BY department;`, + `SELECT range_intersect_agg(multi_event_range) AS intersection_of_multiranges +FROM multiranges;`, + `SELECT range_agg(multi_event_range) AS union_of_multiranges +FROM multiranges;`, + `CREATE OR REPLACE FUNCTION aggregate_ranges() +RETURNS INT4MULTIRANGE AS $$ +DECLARE + aggregated_range INT4MULTIRANGE; +BEGIN + SELECT range_agg(range_value) INTO aggregated_range FROM ranges; + SELECT + department, + any_value(employee_name) AS any_employee + FROM employees + GROUP BY department; + RETURN aggregated_range; +END; +$$ LANGUAGE plpgsql;`, + } + aggregateSqls := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[0], []string{"any_value"}), + }, + sqls[1]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[1], []string{"range_intersect_agg"}), + }, + sqls[2]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[2], []string{"range_agg"}), + }, + sqls[3]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", "SELECT range_agg(range_value) FROM ranges;", []string{"range_agg"}), + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[0], []string{"any_value"}), + }, + } + aggregateSqls[sqls[3]] = modifiedIssuesforPLPGSQL(aggregateSqls[sqls[3]], "FUNCTION", "aggregate_ranges") + + parserIssueDetector := NewParserIssueDetector() + for stmt, expectedIssues := range aggregateSqls { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} func TestRegexFunctionsIssue(t *testing.T) { dmlStmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 9ca3ae3adb..1f18e45651 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -49,6 +49,7 @@ const ( PG_QUERY_JSON_FUNC_EXPR_NODE = "pg_query.JsonFuncExpr" PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" + PG_QUERY_JSON_IS_PREDICATE_NODE = "pg_query.JsonIsPredicate" PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" ) From 5a50ec5c7c86afb2436278557e493c4fa7a9b87a Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Mon, 6 Jan 2025 19:46:37 +0530 Subject: [PATCH 090/105] Reporting Foreign Key references Partitioned table in analyze and assessment (#2145) --- .../dummy-export-dir/schema/tables/table.sql | 2 +- .../tests/analyze-schema/expected_issues.json | 14 +- .../expectedAssessmentReport.json | 14 +- .../pg_assessment_report.sql | 4 +- yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/cmd/exportSchema.go | 5 +- yb-voyager/src/query/queryissue/constants.go | 3 + .../src/query/queryissue/detectors_ddl.go | 20 +++ yb-voyager/src/query/queryissue/issues_ddl.go | 15 +++ .../src/query/queryissue/issues_ddl_test.go | 18 +++ .../queryissue/parser_issue_detector_test.go | 53 ++++++++ .../src/query/queryparser/ddl_processor.go | 126 ++++++++++++------ .../src/query/queryparser/helpers_struct.go | 9 +- .../src/query/queryparser/object_collector.go | 10 +- .../src/query/queryparser/traversal_proto.go | 1 + yb-voyager/src/utils/utils.go | 4 + 17 files changed, 239 insertions(+), 61 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 5464246d5c..9ea26b115f 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -27,7 +27,7 @@ CREATE TABLE sales ( -- cases for multi column list partition, to be reported during analyze-schema CREATE TABLE test_1 ( - id numeric NOT NULL, + id numeric NOT NULL REFERENCES sales_data(sales_id), country_code varchar(3), record_type varchar(5), descriptions varchar(50), diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 0fc61bbf66..6f5af66fbe 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -90,6 +90,16 @@ "GH": "", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "test_1, constraint: (test_1_id_fkey)", + "Reason": "Foreign key constraint references partitioned table", + "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL REFERENCES sales_data(sales_id),\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", + "Suggestion": "No workaround available ", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "MVIEW", @@ -466,7 +476,7 @@ "ObjectType": "TABLE", "ObjectName": "test_1", "Reason": "cannot use \"list\" partition strategy with more than one column", - "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", + "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL REFERENCES sales_data(sales_id),\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", "Suggestion": "Make it a single column partition by list or choose other supported Partitioning methods", "GH": "https://github.com/yugabyte/yb-voyager/issues/699", @@ -1515,7 +1525,7 @@ "ObjectType": "TABLE", "ObjectName": "test_1", "Reason": "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE - (country_code, record_type)", - "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL,\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", + "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL REFERENCES sales_data(sales_id),\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", "Suggestion": "Add all Partition columns to Primary Key", "GH": "https://github.com/yugabyte/yb-voyager/issues/578", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 76e5ffb549..9332320b12 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -45,7 +45,7 @@ { "ObjectType": "TABLE", "TotalCount": 82, - "InvalidCount": 35, + "InvalidCount": 36, "ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { @@ -634,6 +634,16 @@ ], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Foreign key constraint references partitioned table", + "Objects": [ + { + "ObjectName": "public.test_jsonb, constraint: (test_jsonb_id_region_fkey)", + "SqlStatement": "ALTER TABLE ONLY public.test_jsonb\n ADD CONSTRAINT test_jsonb_id_region_fkey FOREIGN KEY (id, region) REFERENCES public.sales_region(id, region);" + } + ], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Regex Functions", "Objects": [ @@ -1011,7 +1021,7 @@ "SchemaName": "public", "ObjectName": "test_jsonb", "RowCount": 0, - "ColumnCount": 3, + "ColumnCount": 4, "Reads": 0, "Writes": 0, "ReadsPerSecond": 0, diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index 7e3c798ead..72e5ea1e74 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -144,7 +144,9 @@ WITH CHECK OPTION; CREATE TABLE public.test_jsonb ( id integer, data jsonb, - data2 text + data2 text, + region text, + FOREIGN KEY (id, region) REFERENCES sales_region(id, region) ); CREATE TABLE public.inet_type ( diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index d0dc86e9ab..f23c13d0bf 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -638,6 +638,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil queryissue.EXCLUSION_CONSTRAINTS, queryissue.DEFERRABLE_CONSTRAINTS, queryissue.PK_UK_ON_COMPLEX_DATATYPE, + queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, } /* TODO: diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index d007fa217f..f6c17cbb4b 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1043,6 +1043,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index 22de944ab8..f84888a8fd 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -95,7 +95,7 @@ func exportSchema() error { log.Errorf("failed to get migration UUID: %v", err) return fmt.Errorf("failed to get migration UUID: %w", err) } - + utils.PrintAndLog("export of schema for source type as '%s'\n", source.DBType) // Check connection with source database. err = source.DB().Connect() @@ -467,8 +467,7 @@ func applyShardingRecommendationIfMatching(sqlInfo *sqlInfo, shardedTables []str } // true -> oracle, false -> PG - parsedObjectName := lo.Ternary(relation.Schemaname == "", relation.Relname, - relation.Schemaname+"."+relation.Relname) + parsedObjectName := utils.BuildObjectName(relation.Schemaname, relation.Relname) match := false switch source.DBType { diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 63c89270e0..e6ab413d27 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -76,6 +76,9 @@ const ( MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" COPY_FROM_WHERE = "COPY FROM ... WHERE" COPY_ON_ERROR = "COPY ... ON_ERROR" + + FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE = "FOREIGN_KEY_REFERENCED_PARTITIONED_TABLE" + FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME = "Foreign key constraint references partitioned table" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index c63a1e9799..7a5401976b 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -95,6 +95,15 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss )) } + if c.ConstraintType == queryparser.FOREIGN_CONSTR_TYPE && d.partitionTablesMap[c.ReferencedTable] { + issues = append(issues, NewForeignKeyReferencesPartitionedTableIssue( + TABLE_OBJECT_TYPE, + table.GetObjectName(), + "", + c.ConstraintName, + )) + } + if c.IsPrimaryKeyORUniqueConstraint() { for _, col := range c.Columns { unsupportedColumnsForTable, ok := d.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()] @@ -444,6 +453,17 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q )) } + if alter.ConstraintType == queryparser.FOREIGN_CONSTR_TYPE && + aid.partitionTablesMap[alter.ConstraintReferencedTable] { + //FK constraint references partitioned table + issues = append(issues, NewForeignKeyReferencesPartitionedTableIssue( + TABLE_OBJECT_TYPE, + alter.GetObjectName(), + "", + alter.ConstraintName, + )) + } + if alter.ConstraintType == queryparser.PRIMARY_CONSTR_TYPE && aid.partitionTablesMap[alter.GetObjectName()] { issues = append(issues, NewAlterTableAddPKOnPartiionIssue( diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index d005a1ef3b..2930b5df4f 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -465,3 +465,18 @@ var securityInvokerViewIssue = issue.Issue{ func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatement string) QueryIssue { return newQueryIssue(securityInvokerViewIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) } + +var foreignKeyReferencesPartitionedTableIssue = issue.Issue{ + Type: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, + TypeName: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, + Suggestion: "No workaround available ", + GH: "", // TODO + DocsLink: "", // TODO +} + +func NewForeignKeyReferencesPartitionedTableIssue(objectType string, objectName string, SqlStatement string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, + } + return newQueryIssue(foreignKeyReferencesPartitionedTableIssue, objectType, objectName, SqlStatement, details) +} diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 709dd369d0..55642bd8e4 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -27,6 +27,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" @@ -273,6 +274,19 @@ func testSecurityInvokerView(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "unrecognized parameter", securityInvokerViewIssue) } +func testForeignKeyReferencesPartitionedTableIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE abc1(id int PRIMARY KEY, val text) PARTITION BY RANGE (id); + CREATE TABLE abc_fk(id int PRIMARY KEY, abc_id INT REFERENCES abc1(id), val text) ;`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `cannot reference partitioned table "abc1"`, foreignKeyReferencesPartitionedTableIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -328,6 +342,10 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "multi range datatype", ybVersion), testMultiRangeDatatypeIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "security invoker view", ybVersion), testSecurityInvokerView) assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "foreign key referenced partitioned table", ybVersion), testForeignKeyReferencesPartitionedTableIssue) + assert.True(t, success) } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 28c251c785..4ad6d7b7b9 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -841,3 +841,56 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { } } } + +func TestForeignKeyReferencesPartitionedTableIssues(t *testing.T) { + requiredDDLs := []string{ + `CREATE TABLE abc1(id int PRIMARY KEY, val text) PARTITION BY RANGE (id);`, + `CREATE TABLE schema1.abc(id int PRIMARY KEY, val text) PARTITION BY RANGE (id);`, + } + stmt1 := `CREATE TABLE abc_fk(id int PRIMARY KEY, abc_id INT REFERENCES abc1(id), val text) ;` + stmt2 := `ALTER TABLE schema1.abc_fk1 +ADD CONSTRAINT fk FOREIGN KEY (abc1_id) +REFERENCES schema1.abc (id); +` + stmt3 := `CREATE TABLE abc_fk ( + id INT PRIMARY KEY, + abc_id INT, + val TEXT, + CONSTRAINT fk_abc FOREIGN KEY (abc_id) REFERENCES abc1(id) +); +` + + stmt4 := `CREATE TABLE schema1.abc_fk(id int PRIMARY KEY, abc_id INT, val text, FOREIGN KEY (abc_id) REFERENCES schema1.abc(id));` + + ddlStmtsWithIssues := map[string][]QueryIssue{ + stmt1: []QueryIssue{ + NewForeignKeyReferencesPartitionedTableIssue(TABLE_OBJECT_TYPE, "abc_fk", stmt1, "abc_fk_abc_id_fkey"), + }, + stmt2: []QueryIssue{ + NewForeignKeyReferencesPartitionedTableIssue(TABLE_OBJECT_TYPE, "schema1.abc_fk1", stmt2, "fk"), + }, + stmt3: []QueryIssue{ + NewForeignKeyReferencesPartitionedTableIssue(TABLE_OBJECT_TYPE, "abc_fk", stmt3, "fk_abc"), + }, + stmt4: []QueryIssue{ + NewForeignKeyReferencesPartitionedTableIssue(TABLE_OBJECT_TYPE, "schema1.abc_fk", stmt4, "abc_fk_abc_id_fkey"), + }, + } + parserIssueDetector := NewParserIssueDetector() + for _, stmt := range requiredDDLs { + err := parserIssueDetector.ParseRequiredDDLs(stmt) + assert.NoError(t, err, "Error parsing required ddl: %s", stmt) + } + for stmt, expectedIssues := range ddlStmtsWithIssues { + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index a14485cd89..4bc488f798 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -23,6 +23,8 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" log "github.com/sirupsen/logrus" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) // Base parser interface @@ -122,7 +124,32 @@ func (tableProcessor *TableProcessor) Process(parseTree *pg_query.ParseResult) ( } // Parse columns and their properties - for _, element := range createTableNode.CreateStmt.TableElts { + tableProcessor.parseTableElts(createTableNode.CreateStmt.TableElts, table) + + if table.IsPartitioned { + + partitionElements := createTableNode.CreateStmt.GetPartspec().GetPartParams() + table.PartitionStrategy = createTableNode.CreateStmt.GetPartspec().GetStrategy() + + for _, partElem := range partitionElements { + if partElem.GetPartitionElem().GetExpr() != nil { + table.IsExpressionPartition = true + } else { + table.PartitionColumns = append(table.PartitionColumns, partElem.GetPartitionElem().GetName()) + } + } + } + + return table, nil +} + +func (tableProcessor *TableProcessor) parseTableElts(tableElts []*pg_query.Node, table *Table) { + /* + Parsing the table elements liek column definitions constraint basically all the things inside the () of CREATE TABLE test(id int, CONSTRAINT Pk PRIMARY KEY (id)....); + storing all the information of columns - name, typename, isArraytype and constraints - constraint name, columns involved, type of constraint, is deferrable or not + + */ + for _, element := range tableElts { if element.GetColumnDef() != nil { if tableProcessor.isGeneratedColumn(element.GetColumnDef()) { table.GeneratedColumns = append(table.GeneratedColumns, element.GetColumnDef().Colname) @@ -172,7 +199,14 @@ func (tableProcessor *TableProcessor) Process(parseTree *pg_query.ParseResult) ( table.Constraints[len(table.Constraints)-1] = lastConstraint } } else { - table.addConstraint(constraint.Contype, []string{colName}, constraint.Conname, false) + /* + table_elts:{column_def:{colname:"abc_id" type_name:{names:{string:{sval:"pg_catalog"}} names:{string:{sval:"int4"}} typemod:-1 + location:45} is_local:true constraints:{constraint:{contype:CONSTR_FOREIGN initially_valid:true pktable:{schemaname:"schema1" + relname:"abc" inh:true relpersistence:"p" location:60} pk_attrs:{string:{sval:"id"}} fk_matchtype:"s" fk_upd_action:"a" fk_del_action:"a" + + In case of FKs there is field called PkTable which has reference table information + */ + table.addConstraint(constraint.Contype, []string{colName}, constraint.Conname, false, constraint.Pktable) } } } @@ -190,33 +224,25 @@ func (tableProcessor *TableProcessor) Process(parseTree *pg_query.ParseResult) ( constraint := element.GetConstraint() conType := element.GetConstraint().Contype columns := parseColumnsFromKeys(constraint.GetKeys()) - if conType == EXCLUSION_CONSTR_TYPE { + switch conType { + case EXCLUSION_CONSTR_TYPE: //In case CREATE DDL has EXCLUDE USING gist(room_id '=', time_range WITH &&) - it will be included in columns but won't have columnDef as its a constraint exclusions := constraint.GetExclusions() //exclusions:{list:{items:{index_elem:{name:"room_id" ordering:SORTBY_DEFAULT nulls_ordering:SORTBY_NULLS_DEFAULT}} //items:{list:{items:{string:{sval:"="}}}}}} columns = tableProcessor.parseColumnsFromExclusions(exclusions) + case FOREIGN_CONSTR_TYPE: + // In case of Foreign key constraint if it is present at the end of table column definition + fkAttrs := constraint.FkAttrs + //CREATE TABLE schema1.abc_fk(id int, abc_id INT, val text, PRIMARY KEY(id), FOREIGN KEY (abc_id) REFERENCES schema1.abc(id)); + //table_elts:{constraint:{contype:CONSTR_FOREIGN initially_valid:true pktable:{schemaname:"schema1" relname:"abc" inh:true relpersistence:"p" location:109} + //fk_attrs:{string:{sval:"abc_id"}} pk_attrs:{string:{sval:"id"}} + columns = parseColumnsFromKeys(fkAttrs) } - table.addConstraint(conType, columns, constraint.Conname, constraint.Deferrable) - - } - } + table.addConstraint(conType, columns, constraint.Conname, constraint.Deferrable, constraint.Pktable) - if table.IsPartitioned { - - partitionElements := createTableNode.CreateStmt.GetPartspec().GetPartParams() - table.PartitionStrategy = createTableNode.CreateStmt.GetPartspec().GetStrategy() - - for _, partElem := range partitionElements { - if partElem.GetPartitionElem().GetExpr() != nil { - table.IsExpressionPartition = true - } else { - table.PartitionColumns = append(table.PartitionColumns, partElem.GetPartitionElem().GetName()) - } } } - - return table, nil } func (tableProcessor *TableProcessor) checkInheritance(createTableNode *pg_query.Node_CreateStmt) bool { @@ -286,14 +312,15 @@ type TableColumn struct { } func (tc *TableColumn) GetFullTypeName() string { - return lo.Ternary(tc.TypeSchema != "", tc.TypeSchema+"."+tc.TypeName, tc.TypeName) + return utils.BuildObjectName(tc.TypeSchema, tc.TypeName) } type TableConstraint struct { - ConstraintType pg_query.ConstrType - ConstraintName string - IsDeferrable bool - Columns []string + ConstraintType pg_query.ConstrType + ConstraintName string + IsDeferrable bool + ReferencedTable string + Columns []string } func (c *TableConstraint) IsPrimaryKeyORUniqueConstraint() bool { @@ -319,8 +346,7 @@ func (c *TableConstraint) generateConstraintName(tableName string) string { } func (t *Table) GetObjectName() string { - qualifiedTable := lo.Ternary(t.SchemaName != "", fmt.Sprintf("%s.%s", t.SchemaName, t.TableName), t.TableName) - return qualifiedTable + return utils.BuildObjectName(t.SchemaName, t.TableName) } func (t *Table) GetSchemaName() string { return t.SchemaName } @@ -345,7 +371,7 @@ func (t *Table) UniqueKeyColumns() []string { return uniqueCols } -func (t *Table) addConstraint(conType pg_query.ConstrType, columns []string, specifiedConName string, deferrable bool) { +func (t *Table) addConstraint(conType pg_query.ConstrType, columns []string, specifiedConName string, deferrable bool, referencedTable *pg_query.RangeVar) { tc := TableConstraint{ ConstraintType: conType, Columns: columns, @@ -354,6 +380,9 @@ func (t *Table) addConstraint(conType pg_query.ConstrType, columns []string, spe generatedConName := tc.generateConstraintName(t.TableName) conName := lo.Ternary(specifiedConName == "", generatedConName, specifiedConName) tc.ConstraintName = conName + if conType == FOREIGN_CONSTR_TYPE { + tc.ReferencedTable = utils.BuildObjectName(referencedTable.Schemaname, referencedTable.Relname) + } t.Constraints = append(t.Constraints, tc) } @@ -404,7 +433,7 @@ type ForeignTable struct { } func (f *ForeignTable) GetObjectName() string { - return lo.Ternary(f.SchemaName != "", f.SchemaName+"."+f.TableName, f.TableName) + return utils.BuildObjectName(f.SchemaName, f.TableName) } func (f *ForeignTable) GetSchemaName() string { return f.SchemaName } @@ -495,7 +524,7 @@ type IndexParam struct { } func (indexParam *IndexParam) GetFullExprCastTypeName() string { - return lo.Ternary(indexParam.ExprCastTypeSchema != "", indexParam.ExprCastTypeSchema+"."+indexParam.ExprCastTypeName, indexParam.ExprCastTypeName) + return utils.BuildObjectName(indexParam.ExprCastTypeSchema, indexParam.ExprCastTypeName) } func (i *Index) GetObjectName() string { @@ -504,7 +533,7 @@ func (i *Index) GetObjectName() string { func (i *Index) GetSchemaName() string { return i.SchemaName } func (i *Index) GetTableName() string { - return lo.Ternary(i.SchemaName != "", fmt.Sprintf("%s.%s", i.SchemaName, i.TableName), i.TableName) + return utils.BuildObjectName(i.SchemaName, i.TableName) } func (i *Index) GetObjectType() string { return INDEX_OBJECT_TYPE } @@ -575,6 +604,15 @@ func (atProcessor *AlterTableProcessor) Process(parseTree *pg_query.ParseResult) alter.IsDeferrable = constraint.Deferrable alter.ConstraintNotValid = constraint.SkipValidation // this is set for the NOT VALID clause alter.ConstraintColumns = parseColumnsFromKeys(constraint.GetKeys()) + if alter.ConstraintType == FOREIGN_CONSTR_TYPE { + /* + alter_table_cmd:{subtype:AT_AddConstraint def:{constraint:{contype:CONSTR_FOREIGN conname:"fk" initially_valid:true + pktable:{schemaname:"schema1" relname:"abc" inh:true relpersistence:"p" + In case of FKs the reference table is in PKTable field and columns are in FkAttrs + */ + alter.ConstraintColumns = parseColumnsFromKeys(constraint.FkAttrs) + alter.ConstraintReferencedTable = utils.BuildObjectName(constraint.Pktable.Schemaname, constraint.Pktable.Relname) + } case pg_query.AlterTableType_AT_DisableRule: /* @@ -604,16 +642,16 @@ type AlterTable struct { NumSetAttributes int NumStorageOptions int //In case AlterType - ADD_CONSTRAINT - ConstraintType pg_query.ConstrType - ConstraintName string - ConstraintNotValid bool - IsDeferrable bool - ConstraintColumns []string + ConstraintType pg_query.ConstrType + ConstraintName string + ConstraintNotValid bool + ConstraintReferencedTable string + IsDeferrable bool + ConstraintColumns []string } func (a *AlterTable) GetObjectName() string { - qualifiedTable := lo.Ternary(a.SchemaName != "", fmt.Sprintf("%s.%s", a.SchemaName, a.TableName), a.TableName) - return qualifiedTable + return utils.BuildObjectName(a.SchemaName, a.TableName) } func (a *AlterTable) GetSchemaName() string { return a.SchemaName } @@ -678,7 +716,7 @@ type Policy struct { } func (p *Policy) GetObjectName() string { - qualifiedTable := lo.Ternary(p.SchemaName != "", fmt.Sprintf("%s.%s", p.SchemaName, p.TableName), p.TableName) + qualifiedTable := utils.BuildObjectName(p.SchemaName, p.TableName) return fmt.Sprintf("%s ON %s", p.PolicyName, qualifiedTable) } func (p *Policy) GetSchemaName() string { return p.SchemaName } @@ -754,7 +792,7 @@ func (t *Trigger) GetObjectName() string { } func (t *Trigger) GetTableName() string { - return lo.Ternary(t.SchemaName != "", fmt.Sprintf("%s.%s", t.SchemaName, t.TableName), t.TableName) + return utils.BuildObjectName(t.SchemaName, t.TableName) } func (t *Trigger) GetSchemaName() string { return t.SchemaName } @@ -831,7 +869,7 @@ type CreateType struct { } func (c *CreateType) GetObjectName() string { - return lo.Ternary(c.SchemaName != "", fmt.Sprintf("%s.%s", c.SchemaName, c.TypeName), c.TypeName) + return utils.BuildObjectName(c.SchemaName, c.TypeName) } func (c *CreateType) GetSchemaName() string { return c.SchemaName } @@ -853,7 +891,7 @@ func (v *ViewProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, err viewSchemaName := viewNode.ViewStmt.View.Schemaname viewName := viewNode.ViewStmt.View.Relname - qualifiedViewName := lo.Ternary(viewSchemaName == "", viewName, viewSchemaName+"."+viewName) + qualifiedViewName := utils.BuildObjectName(viewSchemaName, viewName) /* view_stmt:{view:{schemaname:"public" relname:"invoker_view" inh:true relpersistence:"p" location:12} @@ -886,7 +924,7 @@ type View struct { } func (v *View) GetObjectName() string { - return lo.Ternary(v.SchemaName != "", fmt.Sprintf("%s.%s", v.SchemaName, v.ViewName), v.ViewName) + return utils.BuildObjectName(v.SchemaName, v.ViewName) } func (v *View) GetSchemaName() string { return v.SchemaName } @@ -918,7 +956,7 @@ type MView struct { } func (mv *MView) GetObjectName() string { - return lo.Ternary(mv.SchemaName != "", fmt.Sprintf("%s.%s", mv.SchemaName, mv.ViewName), mv.ViewName) + return utils.BuildObjectName(mv.SchemaName, mv.ViewName) } func (mv *MView) GetSchemaName() string { return mv.SchemaName } diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 37d421bdf9..c3128de6b2 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -20,7 +20,8 @@ import ( "strings" pg_query "github.com/pganalyze/pg_query_go/v6" - "github.com/samber/lo" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) const ( @@ -68,7 +69,7 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string } funcNameList := stmt.GetFuncname() funcSchemaName, funcName := getFunctionObjectName(funcNameList) - return objectType, lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName) + return objectType, utils.BuildObjectName(funcSchemaName, funcName) case isViewStmt: viewName := viewNode.ViewStmt.View return "VIEW", getObjectNameFromRangeVar(viewName) @@ -83,7 +84,7 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string indexName := createIndexNode.IndexStmt.Idxname schemaName := createIndexNode.IndexStmt.Relation.GetSchemaname() tableName := createIndexNode.IndexStmt.Relation.GetRelname() - fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName) + fullyQualifiedName := utils.BuildObjectName(schemaName, tableName) displayObjName := fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName) return "INDEX", displayObjName default: @@ -99,7 +100,7 @@ func isArrayType(typeName *pg_query.TypeName) bool { func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string { schema := obj.Schemaname name := obj.Relname - return lo.Ternary(schema != "", fmt.Sprintf("%s.%s", schema, name), name) + return utils.BuildObjectName(schema, name) } func getFunctionObjectName(funcNameList []*pg_query.Node) (string, string) { diff --git a/yb-voyager/src/query/queryparser/object_collector.go b/yb-voyager/src/query/queryparser/object_collector.go index 4da4682652..ba7561a1b5 100644 --- a/yb-voyager/src/query/queryparser/object_collector.go +++ b/yb-voyager/src/query/queryparser/object_collector.go @@ -20,8 +20,10 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" - "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "google.golang.org/protobuf/reflect/protoreflect" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) // ObjectPredicate defines a function signature that decides whether to include an object. @@ -76,7 +78,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { case PG_QUERY_RANGEVAR_NODE: schemaName := GetStringField(msg, "schemaname") relName := GetStringField(msg, "relname") - objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName) + objectName := utils.BuildObjectName(schemaName, relName) log.Debugf("[RangeVar] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) // it will be either table or view, considering objectType=table for both if c.predicate(schemaName, relName, constants.TABLE) { @@ -89,7 +91,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { if relationMsg != nil { schemaName := GetStringField(relationMsg, "schemaname") relName := GetStringField(relationMsg, "relname") - objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName) + objectName := utils.BuildObjectName(schemaName, relName) log.Debugf("[IUD] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) if c.predicate(schemaName, relName, "table") { c.addObject(objectName) @@ -103,7 +105,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { return } - objectName := lo.Ternary(schemaName != "", schemaName+"."+functionName, functionName) + objectName := utils.BuildObjectName(schemaName, functionName) log.Debugf("[Funccall] fetched schemaname=%s objectname=%s field\n", schemaName, objectName) if c.predicate(schemaName, functionName, constants.FUNCTION) { c.addObject(objectName) diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 1f18e45651..08703dd5bb 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -52,6 +52,7 @@ const ( PG_QUERY_JSON_IS_PREDICATE_NODE = "pg_query.JsonIsPredicate" PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" + PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" ) // function type for processing nodes during traversal diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index ecf178a575..b168d886cd 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -740,3 +740,7 @@ func CheckTools(tools ...string) []string { return missingTools } + +func BuildObjectName(schemaName, objName string) string { + return lo.Ternary(schemaName != "", schemaName+"."+objName, objName) +} From 23db6989e2e26535d497b302cd4eed7dde2bf3a5 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Tue, 7 Jan 2025 17:07:46 +0530 Subject: [PATCH 091/105] Reporting jsonb subscripting issue in assessment and analyze (#2129) Reporting the JSONB subscription in the DMLs, DDLs, and PLPGSQL objects. The approach to detecting these is unclear, so trying to detect as many as possible in the source schema. There are still edge cases that can't be detected easily. https://yugabyte.atlassian.net/browse/DB-14545 --- .../expectedAssessmentReport.json | 81 ++++++++++-- .../pg_assessment_report_uqc.sql | 38 +++++- .../unsupported_query_constructs.sql | 13 +- yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/src/query/queryissue/constants.go | 8 +- yb-voyager/src/query/queryissue/detectors.go | 62 ++++++++++ .../src/query/queryissue/detectors_test.go | 34 +++++ yb-voyager/src/query/queryissue/helpers.go | 23 ++++ yb-voyager/src/query/queryissue/issues_dml.go | 13 ++ .../src/query/queryissue/issues_dml_test.go | 17 ++- .../query/queryissue/parser_issue_detector.go | 41 ++++++ .../queryissue/parser_issue_detector_test.go | 117 +++++++++++++++++- .../src/query/queryparser/ddl_processor.go | 40 ++++++ .../src/query/queryparser/helpers_protomsg.go | 6 + .../src/query/queryparser/helpers_struct.go | 77 ++++++++++++ .../src/query/queryparser/traversal_proto.go | 1 + 16 files changed, 557 insertions(+), 15 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index 276d6f01e5..b15b108395 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -25,15 +25,21 @@ }, { "ObjectType": "TABLE", - "TotalCount": 4, - "InvalidCount": 1, - "ObjectNames": "sales.json_data, analytics.metrics, sales.events, sales.orders" + "TotalCount": 5, + "InvalidCount": 2, + "ObjectNames": "analytics.metrics, sales.orders, sales.test_json_chk, sales.events, sales.json_data" }, { "ObjectType": "VIEW", "TotalCount": 3, "InvalidCount": 3, - "ObjectNames": "sales.event_analysis_view, sales.event_analysis_view2, sales.employ_depart_view" + "ObjectNames": "sales.employ_depart_view, sales.event_analysis_view, sales.event_analysis_view2" + }, + { + "ObjectType": "FUNCTION", + "TotalCount": 1, + "InvalidCount": 1, + "ObjectNames": "sales.get_user_info" } ] }, @@ -43,9 +49,10 @@ "sales.orders", "analytics.metrics", "sales.events", - "sales.json_data" + "sales.json_data", + "sales.test_json_chk" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 4 objects (4 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 5 objects (5 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": null, "NumNodes": 3, "VCPUsPerInstance": 4, @@ -86,6 +93,16 @@ "MinimumVersionsFixedIn": null }, { + "FeatureName": "Jsonb Subscripting", + "Objects": [ + { + "ObjectName": "sales.test_json_chk", + "SqlStatement": "CREATE TABLE sales.test_json_chk (\n id integer,\n name text,\n email text,\n active text,\n data jsonb,\n CONSTRAINT test_json_chk_data_check CHECK ((data['key'::text] \u003c\u003e '{}'::jsonb))\n);" + } + ], + "MinimumVersionsFixedIn": null + }, + { "FeatureName": "Json Type Predicate", "Objects": [ { @@ -98,6 +115,20 @@ ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ + { + "SchemaName": "sales", + "ObjectName": "test_json_chk", + "RowCount": 2, + "ColumnCount": 5, + "Reads": 6, + "Writes": 2, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, { "SchemaName": "sales", "ObjectName": "orders", @@ -182,6 +213,30 @@ "DocsLink": "", "MinimumVersionsFixedIn": null }, + { + "ConstructTypeName": "Jsonb Subscripting", + "Query": "SELECT \n data,\n data[$1] AS name, \n (data[$2]) as active\nFROM sales.test_json_chk", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Jsonb Subscripting", + "Query": "SELECT (sales.get_user_info($1))[$2] AS user_info", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Jsonb Subscripting", + "Query": "SELECT (jsonb_build_object($1, $2, $3, $4, $5, $6) || $7)[$8] AS json_obj", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Jsonb Subscripting", + "Query": "SELECT ($1 :: jsonb)[$2][$3] as b", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "Json Type Predicate", "Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY", @@ -189,5 +244,17 @@ "MinimumVersionsFixedIn": null } ], - "UnsupportedPlPgSqlObjects": null + "UnsupportedPlPgSqlObjects": [ + { + "FeatureName": "Jsonb Subscripting", + "Objects": [ + { + "ObjectType": "FUNCTION", + "ObjectName": "sales.get_user_info", + "SqlStatement": "SELECT\n data,\n data['name'] AS name,\n (data['active']) as active\n FROM sales.test_json_chk;" + } + ], + "MinimumVersionsFixedIn": null + } + ] } \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index b4fe358748..987f4f05b4 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -47,6 +47,42 @@ create view sales.employ_depart_view AS SELECT any_value(name) AS any_employee FROM employees; +CREATE TABLE sales.test_json_chk ( + id int, + name text, + email text, + active text, + data jsonb, + CHECK (data['key']<>'{}') +); + +INSERT INTO sales.test_json_chk (id, name, email, active, data) +VALUES (1, 'John Doe', 'john@example.com', 'Y', jsonb_build_object('key', 'value', 'name', 'John Doe', 'active', 'Y')); + +INSERT INTO sales.test_json_chk (id, name, email, active, data) +VALUES (2, 'Jane Smith', 'jane@example.com', 'N', jsonb_build_object('key', 'value', 'name', 'Jane Smith', 'active', 'N')); + +CREATE OR REPLACE FUNCTION sales.get_user_info(user_id INT) +RETURNS JSONB AS $$ +BEGIN + PERFORM + data, + data['name'] AS name, + (data['active']) as active + FROM sales.test_json_chk; + + RETURN ( + SELECT jsonb_build_object( + 'id', id, + 'name', name, + 'email', email, + 'active', active + ) + FROM sales.test_json_chk + WHERE id = user_id + ); +END; +$$ LANGUAGE plpgsql; CREATE TABLE sales.events ( id int PRIMARY KEY, event_range daterange @@ -85,4 +121,4 @@ INSERT INTO public.json_data ( 3, '[1, 2, 3, 4]', 4, '"hello"', 5, '{"uniqueKey1": "value1", "uniqueKey2": "value2"}' -); \ No newline at end of file +); diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index 43a68b8e37..d2f4f089e3 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -27,6 +27,17 @@ SELECT any_value(name) AS any_employee FROM employees; +SELECT (sales.get_user_info(2))['name'] AS user_info; + +SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 17, 'open_source', TRUE) || '{"key": "value2"}')['name'] AS json_obj; + +SELECT + data, + data['name'] AS name, + (data['active']) as active +FROM sales.test_json_chk; + +SELECT ('{"a": { "b": {"c": "1"}}}' :: jsonb)['a']['b'] as b; --PG15 SELECT range_agg(event_range) AS union_of_ranges FROM sales.events; @@ -37,4 +48,4 @@ FROM sales.events; -- -- PG 16 and above feature SELECT * FROM sales.json_data -WHERE array_column IS JSON ARRAY; \ No newline at end of file +WHERE array_column IS JSON ARRAY; diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f6c17cbb4b..f22eba98e1 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1043,6 +1043,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index e6ab413d27..edb8edf27d 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -73,9 +73,11 @@ const ( FETCH_WITH_TIES = "FETCH_WITH_TIES" REGEX_FUNCTIONS = "REGEX_FUNCTIONS" - MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" - COPY_FROM_WHERE = "COPY FROM ... WHERE" - COPY_ON_ERROR = "COPY ... ON_ERROR" + JSONB_SUBSCRIPTING = "JSONB_SUBSCRIPTING" + JSONB_SUBSCRIPTING_NAME = "Jsonb Subscripting" + MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" + COPY_FROM_WHERE = "COPY FROM ... WHERE" + COPY_ON_ERROR = "COPY ... ON_ERROR" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE = "FOREIGN_KEY_REFERENCED_PARTITIONED_TABLE" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME = "Foreign key constraint references partitioned table" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index d1ec43c5e9..1bc45b0999 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -206,6 +206,68 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type JsonbSubscriptingDetector struct { + query string + jsonbColumns []string + detected bool + jsonbFunctions []string +} + +func NewJsonbSubscriptingDetector(query string, jsonbColumns []string, jsonbFunctions []string) *JsonbSubscriptingDetector { + return &JsonbSubscriptingDetector{ + query: query, + jsonbColumns: jsonbColumns, + jsonbFunctions: jsonbFunctions, + } +} + +func (j *JsonbSubscriptingDetector) Detect(msg protoreflect.Message) error { + + if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_A_INDIRECTION_NODE { + return nil + } + aIndirectionNode, ok := queryparser.GetAIndirectionNode(msg) + if !ok { + return nil + } + + /* + Indirection node is to determine if subscripting is happening in the query e.g. data['name'] - jsonb, numbers[1] - array type, and ('{"a": {"b": {"c": 1}}}'::jsonb)['a']['b']['c']; + Arg is the data on which subscripting is happening e.g data, numbers (columns) and constant data type casted to jsonb ('{"a": {"b": {"c": 1}}}'::jsonb) + Indices are the actual fields that are being accessed while subscripting or the index in case of array type e.g. name, 1, a, b etc. + So we are checking the arg is of jsonb type here + */ + arg := aIndirectionNode.GetArg() + if arg == nil { + return nil + } + /* + Caveats - + + Still with this approach we won't be able to cover all cases e.g. + + select ab_data['name'] from (select Data as ab_data from test_jsonb);`, + + parseTree - stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{a_indirection:{arg:{column_ref:{fields:{string:{sval:"ab_data"}} location:9}} + indirection:{a_indices:{uidx:{a_const:{sval:{sval:"name"} location:17}}}}}} location:9}} from_clause:{range_subselect:{subquery:{select_stmt:{ + target_list:{res_target:{name:"ab_data" val:{column_ref:{fields:{string:{sval:"data"}} location:38}} location:38}} + from_clause:{range_var:{relname:"test_jsonb" inh:true relpersistence:"p" location:59}} limit_option:LIMIT_OPTION_DEFAULT op:SETOP_NONE}}}} + limit_option:LIMIT_OPTION_DEFAULT op:SETOP_NONE}} + */ + if queryparser.DoesNodeHandleJsonbData(arg, j.jsonbColumns, j.jsonbFunctions) { + j.detected = true + } + return nil +} + +func (j *JsonbSubscriptingDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if j.detected { + issues = append(issues, NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", j.query)) + } + return issues +} + type SelectStmtDetector struct { query string limitOptionWithTiesDetected bool diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index e1aa7a32aa..08aaf1b4af 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -680,6 +680,40 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { } } +func TestJsonbSubscriptingDetector(t *testing.T) { + withoutIssueSqls := []string{ + `SELECT numbers[1] AS first_number + FROM array_data;`, + `select ab_data['name'] from (select Data as ab_data from test_jsonb);`, // NOT REPORTED AS OF NOW because of caveat + } + issuesSqls := []string{ + `SELECT ('{"a": {"b": {"c": 1}}}'::jsonb)['a']['b']['c'];`, + `UPDATE json_data +SET data = jsonb_set(data, '{user,details,city}', '"San Francisco"') +WHERE data['user']['name'] = '"Alice"';`, + `SELECT + data->>$1 AS name, + data[$2][$3] AS second_score +FROM test_jsonb1`, + `SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 14, 'open_source', TRUE) || '{"key": "value2"}')['name'] AS json_obj;`, + `SELECT (data || '{"new_key": "new_value"}' )['name'] FROM test_jsonb;`, + `SELECT ('{"key": "value1"}'::jsonb || '{"key": "value2"}'::jsonb)['key'] AS object_in_array;`, + } + + for _, sql := range withoutIssueSqls { + issues := getDetectorIssues(t, NewJsonbSubscriptingDetector(sql, []string{}, []string{}), sql) + + assert.Equal(t, 0, len(issues), "Expected 1 issue for SQL: %s", sql) + } + + for _, sql := range issuesSqls { + issues := getDetectorIssues(t, NewJsonbSubscriptingDetector(sql, []string{"data"}, []string{"jsonb_build_object"}), sql) + + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSONB_SUBSCRIPTING, issues[0].Type, "Expected System Columns issue for SQL: %s", sql) + } +} + func TestJsonConstructorDetector(t *testing.T) { sql := `SELECT JSON_ARRAY('PostgreSQL', 12, TRUE, NULL) AS json_array;` diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index 12f2a2e245..dfc89f1197 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -134,3 +134,26 @@ var unsupportedLargeObjectFunctions = mapset.NewThreadUnsafeSet([]string{ //functions provided by lo extension, refer - https://www.postgresql.org/docs/current/lo.html#LO-RATIONALE "lo_manage", "lo_oid", }...) + +// catalog functions return type jsonb +var catalogFunctionsReturningJsonb = mapset.NewThreadUnsafeSet([]string{ + /* + SELECT + DISTINCT p.proname AS Function_Name + FROM + pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_language l ON p.prolang = l.oid + LEFT JOIN pg_catalog.pg_namespace n ON p.pronamespace = n.oid + WHERE + pg_catalog.pg_function_is_visible(p.oid) AND pg_catalog.pg_get_function_result(p.oid) = 'jsonb' + + ORDER BY Function_Name; + */ + "jsonb_agg", "jsonb_agg_finalfn", "jsonb_agg_strict", "jsonb_array_element", + "jsonb_build_array", "jsonb_build_object", "jsonb_concat", "jsonb_delete", + "jsonb_delete_path", "jsonb_extract_path", "jsonb_in", "jsonb_insert", + "jsonb_object", "jsonb_object_agg", "jsonb_object_agg_finalfn", "jsonb_object_agg_strict", + "jsonb_object_agg_unique", "jsonb_object_agg_unique_strict", "jsonb_object_field", "jsonb_path_query_array", + "jsonb_path_query_array_tz", "jsonb_path_query_first", "jsonb_path_query_first_tz", "jsonb_recv", + "jsonb_set", "jsonb_set_lax", "jsonb_strip_nulls", "to_jsonb", "ts_headline", +}...) diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 0752051de2..e9b8f25923 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -142,6 +142,19 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } +var jsonbSubscriptingIssue = issue.Issue{ + Type: JSONB_SUBSCRIPTING, + TypeName: JSONB_SUBSCRIPTING_NAME, + TypeDescription: "Jsonb subscripting is not supported in YugabyteDB yet", + Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", + GH: "", + DocsLink: "", //TODO +} + +func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(jsonbSubscriptingIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var jsonPredicateIssue = issue.Issue{ Type: JSON_TYPE_PREDICATE, TypeName: JSON_TYPE_PREDICATE_NAME, diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 6825eb1af0..d9a0b8209b 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -25,9 +25,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" - testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func testLOFunctionsIssue(t *testing.T) { @@ -43,6 +43,17 @@ func testLOFunctionsIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "Transaction for catalog table write operation 'pg_largeobject_metadata' not found", loDatatypeIssue) } +func testJsonbSubscriptingIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, `SELECT ('{"a": {"b": {"c": 1}}}'::jsonb)['a']['b']['c'];`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "cannot subscript type jsonb because it is not an array", loDatatypeIssue) +} + func testRegexFunctionsIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -190,7 +201,7 @@ SELECT FROM any_value_ex GROUP BY department;`, -`CREATE TABLE events ( + `CREATE TABLE events ( id SERIAL PRIMARY KEY, event_range daterange ); @@ -264,6 +275,8 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "json query functions", ybVersion), testJsonQueryFunctions) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "json subscripting", ybVersion), testJsonbSubscriptingIssue) + assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "aggregate functions", ybVersion), testAggFunctions) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index e4db26c0fd..defde49879 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -63,6 +63,12 @@ type ParserIssueDetector struct { // Boolean to check if there are any unlogged tables that were filtered // out because they are fixed as per the target db version IsUnloggedTablesIssueFiltered bool + + //Functions in exported schema + functionObjects []*queryparser.Function + + //columns names with jsonb type + jsonbColumns []string } func NewParserIssueDetector() *ParserIssueDetector { @@ -221,6 +227,11 @@ func (p *ParserIssueDetector) ParseRequiredDDLs(query string) error { p.columnsWithUnsupportedIndexDatatypes[table.GetObjectName()][col.ColumnName] = "user_defined_type" } } + + if col.TypeName == "jsonb" { + // used to detect the jsonb subscripting happening on these columns + p.jsonbColumns = append(p.jsonbColumns, col.ColumnName) + } } case *queryparser.CreateType: @@ -235,6 +246,9 @@ func (p *ParserIssueDetector) ParseRequiredDDLs(query string) error { if index.AccessMethod == GIN_ACCESS_METHOD { p.IsGinIndexPresentInSchema = true } + case *queryparser.Function: + fn, _ := ddlObj.(*queryparser.Function) + p.functionObjects = append(p.functionObjects, fn) } return nil } @@ -379,6 +393,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), + NewJsonbSubscriptingDetector(query, p.jsonbColumns, p.getJsonbReturnTypeFunctions()), NewJsonPredicateExprDetector(query), } @@ -423,3 +438,29 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) return result, nil } + +func (p *ParserIssueDetector) getJsonbReturnTypeFunctions() []string { + var jsonbFunctions []string + jsonbColumns := p.jsonbColumns + for _, function := range p.functionObjects { + returnType := function.ReturnType + if strings.HasSuffix(returnType, "%TYPE") { + // e.g. public.table_name.column%TYPE + qualifiedColumn := strings.TrimSuffix(returnType, "%TYPE") + parts := strings.Split(qualifiedColumn, ".") + column := parts[len(parts)-1] + if slices.Contains(jsonbColumns, column) { + jsonbFunctions = append(jsonbFunctions, function.FuncName) + } + } else { + // e.g. public.udt_type, text, trigger, jsonb + parts := strings.Split(returnType, ".") + typeName := parts[len(parts)-1] + if typeName == "jsonb" { + jsonbFunctions = append(jsonbFunctions, function.FuncName) + } + } + } + jsonbFunctions = append(jsonbFunctions, catalogFunctionsReturningJsonb.ToSlice()...) + return jsonbFunctions +} diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 4ad6d7b7b9..e13597625b 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -573,7 +573,7 @@ JSON_TABLE(data, '$.skills[*]' ) ) AS jt;`, `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, -`CREATE TABLE sales.json_data ( + `CREATE TABLE sales.json_data ( id int PRIMARY KEY, array_column TEXT CHECK (array_column IS JSON ARRAY), unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) @@ -654,6 +654,120 @@ JSON_TABLE(data, '$.skills[*]' } } +func TestJsonbSubscriptingIssue(t *testing.T) { + ddlSqls := []string{ + `CREATE TABLE test_jsonb1 ( + id SERIAL PRIMARY KEY, + data JSONB +);`, + `CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT, + email TEXT, + active BOOLEAN +);`, + `CREATE TABLE test_json_chk ( + id int, + data1 jsonb, + CHECK (data1['key']<>'') +);`, + `CREATE OR REPLACE FUNCTION get_user_info(user_id INT) +RETURNS JSONB AS $$ +BEGIN + RETURN ( + SELECT jsonb_build_object( + 'id', id, + 'name', name, + 'email', email, + 'active', active + ) + FROM users + WHERE id = user_id + ); +END; +$$ LANGUAGE plpgsql;`, + } + sqls := []string{ + + `CREATE TABLE test_json_chk ( + id int, + data1 jsonb, + CHECK (data1['key']<>'') +);`, + `SELECT + data->>'name' AS name, + data['scores'][1] AS second_score +FROM test_jsonb1;`, + `SELECT ('[{"key": "value1"}, {"key": "value2"}]'::jsonb)[1]['key'] AS object_in_array; `, + `SELECT (JSON_OBJECT( + 'movie' VALUE JSON_OBJECT('code' VALUE 'P123', 'title' VALUE 'Jaws'), + 'director' VALUE 'Steven Spielberg' +)::JSONB)['movie'] AS nested_json_object;`, + `SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 14, 'open_source', TRUE))['name'] AS json_obj;`, + `SELECT ('{"key": "value1"}'::jsonb || '{"key": "value2"}'::jsonb)['key'] AS object_in_array;`, + `SELECT ('{"key": "value1"}'::jsonb || '{"key": "value2"}')['key'] AS object_in_array;`, + `SELECT (data || '{"new_key": "new_value"}' )['name'] FROM test_jsonb;`, + `SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 14, 'open_source', TRUE))['name'] AS json_obj;`, + `SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 14, 'open_source', TRUE) || '{"key": "value2"}')['name'] AS json_obj;`, + `SELECT (ROW('Alice', 'Smith', 25))['0'] ;`, + `SELECT (get_user_info(2))['name'] AS user_info;`, + } + + stmtsWithExpectedIssues := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewJsonbSubscriptingIssue(TABLE_OBJECT_TYPE, "test_json_chk", sqls[0]), + }, + sqls[1]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[1]), + }, + sqls[2]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[2]), + }, + sqls[3]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[3]), + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[3], []string{JSON_OBJECT}), + }, + sqls[4]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[4]), + }, + sqls[5]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[5]), + }, + sqls[6]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[6]), + }, + sqls[7]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[7]), + }, + sqls[8]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[8]), + }, + sqls[9]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[9]), + }, + sqls[10]: []QueryIssue{}, + sqls[11]: []QueryIssue{ + NewJsonbSubscriptingIssue(DML_QUERY_OBJECT_TYPE, "", sqls[11]), + }, + } + + parserIssueDetector := NewParserIssueDetector() + for _, stmt := range ddlSqls { + err := parserIssueDetector.ParseRequiredDDLs(stmt) + assert.NoError(t, err, "Error parsing required ddl: %s", stmt) + } + for stmt, expectedIssues := range stmtsWithExpectedIssues { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} func TestAggregateFunctions(t *testing.T) { sqls := []string{ `SELECT @@ -710,6 +824,7 @@ $$ LANGUAGE plpgsql;`, } } } + func TestRegexFunctionsIssue(t *testing.T) { dmlStmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 4bc488f798..412dc4c8c8 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -962,6 +962,43 @@ func (mv *MView) GetSchemaName() string { return mv.SchemaName } func (mv *MView) GetObjectType() string { return MVIEW_OBJECT_TYPE } +// ============================Function Processor ================= + +type FunctionProcessor struct{} + +func NewFunctionProcessor() *FunctionProcessor { + return &FunctionProcessor{} +} + +func (mv *FunctionProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + funcNode, ok := getCreateFuncStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE FUNCTION statement") + } + + funcNameList := funcNode.CreateFunctionStmt.GetFuncname() + funcSchemaName, funcName := getFunctionObjectName(funcNameList) + function := Function{ + SchemaName: funcSchemaName, + FuncName: funcName, + ReturnType: GetReturnTypeOfFunc(parseTree), + } + return &function, nil +} + +type Function struct { + SchemaName string + FuncName string + ReturnType string +} + +func (f *Function) GetObjectName() string { + return lo.Ternary(f.SchemaName != "", fmt.Sprintf("%s.%s", f.SchemaName, f.FuncName), f.FuncName) +} +func (f *Function) GetSchemaName() string { return f.SchemaName } + +func (f *Function) GetObjectType() string { return FUNCTION_OBJECT_TYPE } + //=============================No-Op PROCESSOR ================== //No op Processor for objects we don't have Processor yet @@ -1010,6 +1047,8 @@ func GetDDLProcessor(parseTree *pg_query.ParseResult) (DDLProcessor, error) { } else { return NewNoOpProcessor(), nil } + case PG_QUERY_CREATE_FUNCTION_STMT: + return NewFunctionProcessor(), nil default: return NewNoOpProcessor(), nil } @@ -1046,6 +1085,7 @@ const ( PG_QUERY_FOREIGN_TABLE_STMT = "pg_query.CreateForeignTableStmt" PG_QUERY_VIEW_STMT = "pg_query.ViewStmt" PG_QUERY_CREATE_TABLE_AS_STMT = "pg_query.CreateTableAsStmt" + PG_QUERY_CREATE_FUNCTION_STMT = "pg_query.CreateFunctionStmt" ) var deferrableConstraintsList = []pg_query.ConstrType{ diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index de3204f3c9..73d76b7e1b 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -449,3 +449,9 @@ func TraverseAndExtractDefNamesFromDefElem(msg protoreflect.Message) ([]string, return defNames, nil } + +func GetAIndirectionNode(msg protoreflect.Message) (*pg_query.A_Indirection, bool) { + protoMsg := msg.Interface().(protoreflect.ProtoMessage) + aIndirection, ok := protoMsg.(*pg_query.A_Indirection) + return aIndirection, ok +} diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index c3128de6b2..1efbe99dac 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -17,6 +17,7 @@ package queryparser import ( "fmt" + "slices" "strings" pg_query "github.com/pganalyze/pg_query_go/v6" @@ -216,6 +217,9 @@ func getQualifiedTypeName(typeNames []*pg_query.Node) string { } func convertParserTypeNameToString(typeVar *pg_query.TypeName) string { + if typeVar == nil { + return "" + } typeNames := typeVar.GetNames() finalTypeName := getQualifiedTypeName(typeNames) // type name can qualified table_name.column in case of %TYPE if typeVar.PctType { // %TYPE declaration, so adding %TYPE for using it further @@ -265,3 +269,76 @@ func IsDDL(parseTree *pg_query.ParseResult) (bool, error) { //Not Full-proof as we don't have all DDL types but atleast we will skip all the types we know currently return !ok, nil } + +/* +this function checks whether the current node handles the jsonb data or not by evaluating all different type of nodes - +column ref - column of jsonb type +type cast - constant data with type casting to jsonb type +func call - function call returning the jsonb data +Expression - if any of left and right operands are of node type handling jsonb data +*/ +func DoesNodeHandleJsonbData(node *pg_query.Node, jsonbColumns []string, jsonbFunctions []string) bool { + switch { + case node.GetColumnRef() != nil: + /* + SELECT numbers[1] AS first_number + FROM array_data; + {a_indirection:{arg:{column_ref:{fields:{string:{sval:"numbers"}} location:69}} + indirection:{a_indices:{uidx:{a_const:{ival:{ival:1} location:77}}}}}} location:69}} + */ + _, col := GetColNameFromColumnRef(node.GetColumnRef().ProtoReflect()) + if slices.Contains(jsonbColumns, col) { + return true + } + + case node.GetTypeCast() != nil: + /* + SELECT ('{"a": {"b": {"c": 1}}}'::jsonb)['a']['b']['c']; + {a_indirection:{arg:{type_cast:{arg:{a_const:{sval:{sval:"{\"a\": {\"b\": {\"c\": 1}}}"} location:280}} + type_name:{names:{string:{sval:"jsonb"}} typemod:-1 location:306} location:304}} + */ + typeCast := node.GetTypeCast() + typeName, _ := getTypeNameAndSchema(typeCast.GetTypeName().GetNames()) + if typeName == "jsonb" { + return true + } + case node.GetFuncCall() != nil: + /* + SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 14, 'open_source', TRUE))['name'] AS json_obj; + val:{a_indirection:{arg:{func_call:{funcname:{string:{sval:"jsonb_build_object"}} args:{a_const:{sval:{sval:"name"} + location:194}} args:{a_const:{sval:{sval:"PostgreSQL"} location:202}} args:{a_const:{sval:{sval:"version"} location:216}} + args:{a_const:{ival:{ival:14} location:227}} + */ + funcCall := node.GetFuncCall() + _, funcName := getFunctionObjectName(funcCall.Funcname) + if slices.Contains(jsonbFunctions, funcName) { + return true + } + case node.GetAExpr() != nil: + /* + SELECT ('{"key": "value1"}'::jsonb || '{"key1": "value2"}'::jsonb)['key'] AS object_in_array; + val:{a_indirection:{arg:{a_expr:{kind:AEXPR_OP name:{string:{sval:"||"}} lexpr:{type_cast:{arg:{a_const:{sval:{sval:"{\"key\": \"value1\"}"} + location:81}} type_name:{names:{string:{sval:"jsonb"}} typemod:-1 location:102} location:100}} rexpr:{type_cast:{arg:{a_const:{sval:{sval:"{\"key1\": \"value2\"}"} + location:111}} type_name:{names:{string:{sval:"jsonb"}} typemod:-1 location:132} location:130}} location:108}} indirection:{a_indices:{uidx:{a_const:{sval:{sval:"key"} + location:139}}}}}} + + SELECT (data || '{"new_key": "new_value"}' )['name'] FROM test_jsonb; + {val:{a_indirection:{arg:{a_expr:{kind:AEXPR_OP name:{string:{sval:"||"}} lexpr:{column_ref:{fields:{string:{sval:"data"}} location:10}} rexpr:{a_const:{sval:{sval:"{\"new_key\": \"new_value\"}"} + location:18}} location:15}} indirection:{a_indices:{uidx:{a_const:{sval:{sval:"name"} + + SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 14, 'open_source', TRUE) || '{"key": "value2"}')['name'] AS json_obj; + {val:{a_indirection:{arg:{a_expr:{kind:AEXPR_OP name:{string:{sval:"||"}} lexpr:{column_ref:{fields:{string:{sval:"data"}} location:10}} rexpr:{a_const:{sval:{sval:"{\"new_key\": \"new_value\"}"} + location:18}} location:15}} indirection:{a_indices:{uidx:{a_const:{sval:{sval:"name"} location:47}}}}}} location:9}} + */ + expr := node.GetAExpr() + lExpr := expr.GetLexpr() + rExpr := expr.GetRexpr() + if lExpr != nil && DoesNodeHandleJsonbData(lExpr, jsonbColumns, jsonbFunctions) { + return true + } + if rExpr != nil && DoesNodeHandleJsonbData(rExpr, jsonbColumns, jsonbFunctions) { + return true + } + } + return false +} diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 08703dd5bb..9177afc10c 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -43,6 +43,7 @@ const ( PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" PG_QUERY_SELECTSTMT_NODE = "pg_query.SelectStmt" + PG_QUERY_A_INDIRECTION_NODE = "pg_query.A_Indirection" PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE = "pg_query.JsonArrayConstructor" From 2cccc27407ace0506d3b18dc13314f4bb41e410e Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:22:35 +0530 Subject: [PATCH 092/105] Added a test to detect COPY ON ERROR clause in the migration assessment test (#2157) --- .../expectedAssessmentReport.json | 37 +++++++++++++++---- .../unsupported_query_constructs.sql | 27 ++++++++------ 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 9332320b12..c341d8b26a 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 43, + "TotalCount": 42, "InvalidCount": 0, - "ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees_employee_id_seq, public.employeesforview_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.employeesforview_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 82, + "TotalCount": 83, "InvalidCount": 36, - "ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -172,7 +172,8 @@ "public.library_nested", "public.orders_lateral", "public.employees", - "public.employees3", + "public.employeescopyfromwhere", + "public.employeescopyonerror", "public.bigint_multirange_table", "public.date_multirange_table", "public.int_multirange_table", @@ -187,7 +188,7 @@ "schema2.timestamptz_multirange_table", "public.employeesforview" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 89 objects (81 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -2111,7 +2112,7 @@ }, { "SchemaName": "public", - "ObjectName": "employees3", + "ObjectName": "employeescopyfromwhere", "RowCount": 2, "ColumnCount": 3, "Reads": 0, @@ -2304,6 +2305,20 @@ "ObjectType": "", "ParentTableName": null, "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "employeescopyonerror", + "RowCount": 3, + "ColumnCount": 3, + "Reads": 0, + "Writes": 3, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 } ], @@ -2507,7 +2522,13 @@ }, { "ConstructTypeName": "COPY FROM ... WHERE", - "Query": "COPY employees3 (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", + "Query": "COPY employeesCopyFromWhere (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "COPY ... ON_ERROR", + "Query": "COPY employeesCopyOnError (id, name, age)\nFROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE )", "DocsLink": "", "MinimumVersionsFixedIn": null } diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index 284c8f7d01..0d1a88f6e6 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -142,15 +142,15 @@ SELECT lo_create('32142'); -- Unsupported COPY constructs -CREATE TABLE IF NOT EXISTS employees3 ( - id SERIAL PRIMARY KEY, +CREATE TABLE IF NOT EXISTS employeesCopyFromWhere ( + id INT PRIMARY KEY, name TEXT NOT NULL, age INT NOT NULL ); -- COPY FROM with WHERE clause -COPY employees3 (id, name, age) +COPY employeesCopyFromWhere (id, name, age) FROM STDIN WITH (FORMAT csv) WHERE age > 30; 1,John Smith,25 @@ -158,12 +158,17 @@ WHERE age > 30; 3,Bob Johnson,31 \. --- This can be uncommented when we start using PG 17 or later in the tests --- -- COPY with ON_ERROR clause --- COPY employees (id, name, age) --- FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE ); --- 4,Adam Smith,22 --- 5,John Doe,34 --- 6,Ron Johnson,31 --- \. +CREATE TABLE IF NOT EXISTS employeesCopyOnError ( + id INT PRIMARY KEY, + name TEXT NOT NULL, + age INT NOT NULL +); + +-- COPY with ON_ERROR clause +COPY employeesCopyOnError (id, name, age) +FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE ); +4,Adam Smith,22 +5,John Doe,34 +6,Ron Johnson,31 +\. From d490424ff9fa2185dcc4d49ac18595d13e2cb189 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 8 Jan 2025 14:08:20 +0530 Subject: [PATCH 093/105] Refactor migration assessment codepath: flatten reported issues (#2153) * Added new arg - impact to reportCase() for assigning impact to the issues detected using regexp in analyze schema * Use Issue Type for issues lookup in schema analysis report wherever possible * Defining impact for issues_ddl and issues_dml and propagating that to AssessmentIssue * moved pg/omnibus schema-migration test to the correct place * Renamed TypeName and TypeDescription field of Issues structs to Name and Description --- .github/workflows/misc-migtests.yml | 3 - .github/workflows/pg-17-migtests.yml | 4 + migtests/scripts/functions.sh | 21 +- yb-voyager/cmd/analyzeSchema.go | 129 +++++----- yb-voyager/cmd/assessMigrationCommand.go | 224 ++++++++++++++---- yb-voyager/cmd/common.go | 16 +- yb-voyager/cmd/constants.go | 75 +++--- yb-voyager/src/constants/constants.go | 16 +- yb-voyager/src/issue/issue.go | 38 ++- yb-voyager/src/query/queryissue/issues_ddl.go | 141 ++++++----- yb-voyager/src/query/queryissue/issues_dml.go | 170 +++++++------ .../src/query/queryparser/object_collector.go | 2 +- yb-voyager/src/utils/commonVariables.go | 1 + 13 files changed, 550 insertions(+), 290 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 3385743b8b..3362028481 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -114,9 +114,6 @@ jobs: echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - echo "TEST: PG sample schemas (omnibus)" - migtests/scripts/run-schema-migration.sh pg/omnibus - echo "Setup the gcp credentials" migtests/scripts/gcs/create_gcs_credentials_file diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index fc2aba70b3..27aacc0c68 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -117,6 +117,10 @@ jobs: if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-schema-migration.sh pg/osm + - name: "TEST: PG sample schemas (omnibus)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/omnibus + - name: "TEST: PG sample schemas (adventureworks)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-schema-migration.sh pg/adventureworks diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 18f3ff9a28..4d8e1d6f4f 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -877,7 +877,13 @@ normalize_json() { # Normalize JSON with jq; use --sort-keys to avoid the need to keep the same sequence of keys in expected vs actual json jq --sort-keys 'walk( if type == "object" then - .ObjectNames? |= (if type == "string" then split(", ") | sort | join(", ") else . end) | + .ObjectNames? |= ( + if type == "string" then + split(", ") | sort | join(", ") + else + . + end + ) | .VoyagerVersion? = "IGNORED" | .TargetDBVersion? = "IGNORED" | .DbVersion? = "IGNORED" | @@ -885,10 +891,17 @@ normalize_json() { .OptimalSelectConnectionsPerNode? = "IGNORED" | .OptimalInsertConnectionsPerNode? = "IGNORED" | .RowCount? = "IGNORED" | - .SqlStatement? |= (if type == "string" then gsub("\\n"; " ") else . end) + # Replace newline characters in SqlStatement with spaces + .SqlStatement? |= ( + if type == "string" then + gsub("\\n"; " ") + else + . + end + ) elif type == "array" then - sort_by(tostring) - else + sort_by(tostring) + else . end )' "$input_file" > "$temp_file" diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index f23c13d0bf..3eb78dc3b5 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -34,6 +34,7 @@ import ( "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryissue" @@ -220,7 +221,7 @@ const ( ) // Reports one case in JSON -func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, issueType string, docsLink string) { +func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, category string, docsLink string, impact string) { var issue utils.AnalyzeSchemaIssue issue.FilePath = filePath issue.Reason = reason @@ -229,7 +230,8 @@ func reportCase(filePath string, reason string, ghIssue string, suggestion strin issue.ObjectType = objType issue.ObjectName = objName issue.SqlStatement = sqlStmt - issue.IssueType = issueType + issue.IssueType = category // IssueType field of analyze schema should be renamed to Category + issue.Impact = lo.Ternary(impact != "", impact, constants.IMPACT_LEVEL_1) if sourceDBType == POSTGRESQL { issue.DocsLink = docsLink } @@ -239,13 +241,13 @@ func reportCase(filePath string, reason string, ghIssue string, suggestion strin func reportBasedOnComment(comment int, fpath string, issue string, suggestion string, objName string, objType string, line string) { if comment == 1 { - reportCase(fpath, "Unsupported, please edit to match PostgreSQL syntax", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "Unsupported, please edit to match PostgreSQL syntax", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap[objType].invalidCount[objName] = true } else if comment == 2 { - // reportCase(fpath, "PACKAGE in oracle are exported as Schema, please review and edit to match PostgreSQL syntax if required, Package is "+objName, issue, suggestion, objType) + // reportCase(fpath, "PACKAGE in oracle are exported as Schema, please review and edit to match PostgreSQL syntax if required, Package is "+objName, issue, suggestion, objType, "") summaryMap["PACKAGE"].objSet = append(summaryMap["PACKAGE"].objSet, objName) } else if comment == 3 { - reportCase(fpath, "SQLs in file might be unsupported please review and edit to match PostgreSQL syntax if required. ", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "SQLs in file might be unsupported please review and edit to match PostgreSQL syntax if required. ", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if comment == 4 { summaryMap[objType].details["Inherited Types are present which are not supported in PostgreSQL syntax, so exported as Inherited Tables"] = true } @@ -321,7 +323,7 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { if !summaryMap[objType].invalidCount[sqlStmtInfo.objName] { reason := fmt.Sprintf("%s - '%s'", UNSUPPORTED_PG_SYNTAX, err.Error()) reportCase(fpath, reason, "https://github.com/yugabyte/yb-voyager/issues/1625", - "Fix the schema as per PG syntax", objType, sqlStmtInfo.objName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "Fix the schema as per PG syntax", objType, sqlStmtInfo.objName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } continue } @@ -348,7 +350,7 @@ func checkViews(sqlInfoArr []sqlInfo, fpath string) { if view := viewWithCheckRegex.FindStringSubmatch(sqlInfo.stmt); view != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, VIEW_CHECK_OPTION_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/22716", - "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "VIEW", view[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, VIEW_CHECK_OPTION_DOC_LINK) + "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "VIEW", view[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, VIEW_CHECK_OPTION_DOC_LINK, "") } } } @@ -372,47 +374,47 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { if rangeRegex.MatchString(sqlInfo.stmt) { reportCase(fpath, "RANGE with offset PRECEDING/FOLLOWING is not supported for column type numeric and offset type double precision", - "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true } else if stmt := fetchRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { location := strings.ToUpper(stmt[1]) if slices.Contains(notSupportedFetchLocation, location) { summaryMap["PROCEDURE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "This FETCH clause might not be supported yet", "https://github.com/YugaByte/yugabyte-db/issues/6514", - "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } else if stmt := alterAggRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { summaryMap["AGGREGATE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER AGGREGATE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropCollRegex.MatchString(sqlInfo.stmt) { summaryMap["COLLATION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropIdxRegex.MatchString(sqlInfo.stmt) { summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropViewRegex.MatchString(sqlInfo.stmt) { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropSeqRegex.MatchString(sqlInfo.stmt) { summaryMap["SEQUENCE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropForeignRegex.MatchString(sqlInfo.stmt) { summaryMap["FOREIGN TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if idx := dropIdxConcurRegex.FindStringSubmatch(sqlInfo.stmt); idx != nil { summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP INDEX CONCURRENTLY not supported yet", - "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if currentOfRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "WHERE CURRENT OF not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/737", "", "CURSOR", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "WHERE CURRENT OF not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/737", "", "CURSOR", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if bulkCollectRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "BULK COLLECT keyword of oracle is not converted into PostgreSQL compatible syntax", "https://github.com/yugabyte/yb-voyager/issues/1539", "", "", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "BULK COLLECT keyword of oracle is not converted into PostgreSQL compatible syntax", "https://github.com/yugabyte/yb-voyager/issues/1539", "", "", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } } @@ -424,106 +426,106 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { if am := amRegex.FindStringSubmatch(sqlInfo.stmt); am != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE ACCESS METHOD is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10693", "", "ACCESS METHOD", am[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10693", "", "ACCESS METHOD", am[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := idxConcRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "REINDEX is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10267", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10267", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := likeAllRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE ALL is not supported yet.", - "https://github.com/yugabyte/yugabyte-db/issues/10697", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10697", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := likeRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE clause not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := withOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "OIDs are not supported for user tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE OF not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterSchemaRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET SCHEMA not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if createSchemaRegex.MatchString(sqlInfo.stmt) { summaryMap["SCHEMA"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE SCHEMA with elements not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterNotOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE NOT OF not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterColumnStatsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STATISTICS not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterColumnStorageRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STORAGE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterColumnResetAttributesRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column RESET (attribute) not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER CONSTRAINT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := setOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITH OIDS not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := withoutClusterRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITHOUT CLUSTER not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterSetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterIdxRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER INDEX SET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterResetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE RESET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterOptionsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if typ := dropAttrRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE DROP ATTRIBUTE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if typ := alterTypeRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterInhRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE INHERIT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := valConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE VALIDATE CONSTRAINT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if spc := alterTblSpcRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { summaryMap["TABLESPACE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLESPACE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if spc := alterViewRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER VIEW not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := cLangRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { reportCase(fpath, "LANGUAGE C not supported yet.", - "https://github.com/yugabyte/yb-voyager/issues/1540", "", "FUNCTION", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yb-voyager/issues/1540", "", "FUNCTION", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap["FUNCTION"].invalidCount[sqlInfo.objName] = true } else if strings.Contains(strings.ToLower(sqlInfo.stmt), "drop temporary table") { filePath := strings.Split(fpath, "/") @@ -531,22 +533,22 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { objType := strings.ToUpper(strings.Split(fileName, ".")[0]) summaryMap[objType].invalidCount[sqlInfo.objName] = true reportCase(fpath, `temporary table is not a supported clause for drop`, - "https://github.com/yugabyte/yb-voyager/issues/705", `remove "temporary" and change it to "drop table"`, objType, sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, DROP_TEMP_TABLE_DOC_LINK) + "https://github.com/yugabyte/yb-voyager/issues/705", `remove "temporary" and change it to "drop table"`, objType, sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, DROP_TEMP_TABLE_DOC_LINK, "") } else if regMatch := anydataRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyData datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyData datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "AnyData datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyData datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := anydatasetRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyDataSet datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyDataSet datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := anyTypeRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "AnyType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := uriTypeRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "URIType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with URIType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "URIType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with URIType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := jsonFuncRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap[objType].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "JSON_ARRAYAGG() function is not available in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1542", `Rename the function to YugabyteDB's equivalent JSON_AGG()`, objType, regMatch[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "JSON_ARRAYAGG() function is not available in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1542", `Rename the function to YugabyteDB's equivalent JSON_AGG()`, objType, regMatch[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } @@ -559,11 +561,11 @@ func checkForeign(sqlInfoArr []sqlInfo, fpath string) { if tbl := primRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Primary key constraints are not supported on foreign tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := foreignKeyRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Foreign key constraints are not supported on foreign tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } } @@ -573,7 +575,7 @@ func checkRemaining(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { if trig := compoundTrigRegex.FindStringSubmatch(sqlInfo.stmt); trig != nil { reportCase(fpath, COMPOUND_TRIGGER_ISSUE_REASON, - "https://github.com/yugabyte/yb-voyager/issues/1543", "", "TRIGGER", trig[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yb-voyager/issues/1543", "", "TRIGGER", trig[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap["TRIGGER"].invalidCount[sqlInfo.objName] = true } } @@ -620,18 +622,18 @@ var MigrationCaveatsIssues = []string{ } func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.AnalyzeSchemaIssue { - issueType := UNSUPPORTED_FEATURES + issueType := UNSUPPORTED_FEATURES_CATEGORY switch true { case isPlPgSQLIssue: - issueType = UNSUPPORTED_PLPGSQL_OBEJCTS + issueType = UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { //Adding the MIGRATION_CAVEATS issueType of the utils.Issue for these issueInstances in MigrationCaveatsIssues - return strings.Contains(issueInstance.TypeName, i) + return strings.Contains(issueInstance.Name, i) }): - issueType = MIGRATION_CAVEATS - case strings.HasPrefix(issueInstance.TypeName, UNSUPPORTED_DATATYPE): + issueType = MIGRATION_CAVEATS_CATEGORY + case strings.HasPrefix(issueInstance.Name, UNSUPPORTED_DATATYPE): //Adding the UNSUPPORTED_DATATYPES issueType of the utils.Issue for these issues whose TypeName starts with "Unsupported datatype ..." - issueType = UNSUPPORTED_DATATYPES + issueType = UNSUPPORTED_DATATYPES_CATEGORY } var constraintIssues = []string{ @@ -674,8 +676,9 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil IssueType: issueType, ObjectType: issueInstance.ObjectType, ObjectName: displayObjectName, - Reason: issueInstance.TypeName, + Reason: issueInstance.Name, Type: issueInstance.Type, + Impact: issueInstance.Impact, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, @@ -690,7 +693,7 @@ func checkExtensions(sqlInfoArr []sqlInfo, fpath string) { if sqlInfo.objName != "" && !slices.Contains(supportedExtensionsOnYB, sqlInfo.objName) { summaryMap["EXTENSION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, UNSUPPORTED_EXTENSION_ISSUE+" Refer to the docs link for the more information on supported extensions.", "https://github.com/yugabyte/yb-voyager/issues/1538", "", "EXTENSION", - sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, EXTENSION_DOC_LINK) + sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, EXTENSION_DOC_LINK, constants.IMPACT_LEVEL_3) } if strings.ToLower(sqlInfo.objName) == "hll" { summaryMap["EXTENSION"].details[`'hll' extension is supported in YugabyteDB v2.18 onwards. Please verify this extension as per the target YugabyteDB version.`] = true @@ -1110,12 +1113,12 @@ func checkConversions(sqlInfoArr []sqlInfo, filePath string) { convName = fmt.Sprintf("%s.%s", convName, nameList[1].GetString_().Sval) } reportCase(filePath, CONVERSION_ISSUE_REASON, "https://github.com/yugabyte/yugabyte-db/issues/10866", - "Remove it from the exported schema", "CONVERSION", convName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK) + "Remove it from the exported schema", "CONVERSION", convName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, CREATE_CONVERSION_DOC_LINK, constants.IMPACT_LEVEL_3) } else { //pg_query doesn't seem to have a Node type of AlterConversionStmt so using regex for now if stmt := alterConvRegex.FindStringSubmatch(sqlStmtInfo.stmt); stmt != nil { reportCase(filePath, "ALTER CONVERSION is not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/10866", - "Remove it from the exported schema", "CONVERSION", stmt[1], sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK) + "Remove it from the exported schema", "CONVERSION", stmt[1], sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, CREATE_CONVERSION_DOC_LINK, "") } } diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f22eba98e1..a80de6b4e9 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -38,6 +38,7 @@ import ( "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" @@ -423,6 +424,7 @@ func assessMigration() (err error) { return fmt.Errorf("failed to generate assessment report: %w", err) } + log.Infof("number of assessment issues detected: %d\n", len(assessmentReport.Issues)) utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() controlPlane.MigrationAssessmentCompleted(completedEvent) @@ -538,8 +540,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment } for _, unsupportedDataType := range ar.UnsupportedDataTypes { issues = append(issues, AssessmentIssueYugabyteD{ - Type: DATATYPE, - TypeDescription: DATATYPE_ISSUE_TYPE_DESCRIPTION, + Type: constants.DATATYPE, + TypeDescription: GetCategoryDescription(constants.DATATYPE), Subtype: unsupportedDataType.DataType, ObjectName: fmt.Sprintf("%s.%s.%s", unsupportedDataType.SchemaName, unsupportedDataType.TableName, unsupportedDataType.ColumnName), SqlStatement: "", @@ -550,8 +552,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, unsupportedFeature := range ar.UnsupportedFeatures { for _, object := range unsupportedFeature.Objects { issues = append(issues, AssessmentIssueYugabyteD{ - Type: FEATURE, - TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, + Type: constants.FEATURE, + TypeDescription: GetCategoryDescription(constants.FEATURE), Subtype: unsupportedFeature.FeatureName, SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features ObjectName: object.ObjectName, @@ -565,8 +567,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, migrationCaveat := range ar.MigrationCaveats { for _, object := range migrationCaveat.Objects { issues = append(issues, AssessmentIssueYugabyteD{ - Type: MIGRATION_CAVEATS, - TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, + Type: constants.MIGRATION_CAVEATS, + TypeDescription: GetCategoryDescription(constants.MIGRATION_CAVEATS), Subtype: migrationCaveat.FeatureName, SubtypeDescription: migrationCaveat.FeatureDescription, ObjectName: object.ObjectName, @@ -579,8 +581,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, uqc := range ar.UnsupportedQueryConstructs { issues = append(issues, AssessmentIssueYugabyteD{ - Type: QUERY_CONSTRUCT, - TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, + Type: constants.QUERY_CONSTRUCT, + TypeDescription: GetCategoryDescription(constants.QUERY_CONSTRUCT), Subtype: uqc.ConstructTypeName, SqlStatement: uqc.Query, DocsLink: uqc.DocsLink, @@ -591,8 +593,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { for _, object := range plpgsqlObjects.Objects { issues = append(issues, AssessmentIssueYugabyteD{ - Type: PLPGSQL_OBJECT, - TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, + Type: constants.PLPGSQL_OBJECT, + TypeDescription: GetCategoryDescription(constants.PLPGSQL_OBJECT), Subtype: plpgsqlObjects.FeatureName, SubtypeDescription: plpgsqlObjects.FeatureDescription, ObjectName: object.ObjectName, @@ -888,7 +890,11 @@ func generateAssessmentReport() (err error) { return fmt.Errorf("failed to fetch columns with unsupported data types: %w", err) } assessmentReport.UnsupportedDataTypes = unsupportedDataTypes - assessmentReport.UnsupportedDataTypesDesc = DATATYPE_ISSUE_TYPE_DESCRIPTION + assessmentReport.UnsupportedDataTypesDesc = DATATYPE_CATEGORY_DESCRIPTION + + assessmentReport.AppendIssues(getAssessmentIssuesForUnsupportedDatatypes(unsupportedDataTypes)...) + + addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) assessmentReport.Sizing = migassessment.SizingReport assessmentReport.TableIndexStats, err = assessmentDB.FetchAllStats() @@ -897,7 +903,6 @@ func generateAssessmentReport() (err error) { } addNotesToAssessmentReport() - addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) postProcessingOfAssessmentReport() assessmentReportDir := filepath.Join(exportDir, "assessment", "reports") @@ -915,20 +920,16 @@ func generateAssessmentReport() (err error) { func getAssessmentReportContentFromAnalyzeSchema() error { /* - Here we are generating analyze schema report which converts issue instance to analyze schema issue - Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) + Here we are generating analyze schema report which converts issue instance to analyze schema issue + Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) - But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). + But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). */ schemaAnalysisReport := analyzeSchemaInternal(&source, true) assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary - assessmentReport.SchemaSummary.Description = SCHEMA_SUMMARY_DESCRIPTION - if source.DBType == ORACLE { - assessmentReport.SchemaSummary.Description = SCHEMA_SUMMARY_DESCRIPTION_ORACLE - } + assessmentReport.SchemaSummary.Description = lo.Ternary(source.DBType == ORACLE, SCHEMA_SUMMARY_DESCRIPTION_ORACLE, SCHEMA_SUMMARY_DESCRIPTION) - // fetching unsupportedFeaturing with the help of Issues report in SchemaReport var unsupportedFeatures []UnsupportedFeature var err error switch source.DBType { @@ -940,13 +941,14 @@ func getAssessmentReportContentFromAnalyzeSchema() error { panic(fmt.Sprintf("unsupported source db type %q", source.DBType)) } if err != nil { - return fmt.Errorf("failed to fetch %s unsupported features: %w", source.DBType, err) + return fmt.Errorf("failed to fetch '%s' unsupported features: %w", source.DBType, err) } assessmentReport.UnsupportedFeatures = append(assessmentReport.UnsupportedFeatures, unsupportedFeatures...) - assessmentReport.UnsupportedFeaturesDesc = FEATURE_ISSUE_TYPE_DESCRIPTION + assessmentReport.UnsupportedFeaturesDesc = FEATURE_CATEGORY_DESCRIPTION + + // Ques: Do we still need this and REPORT_UNSUPPORTED_QUERY_CONSTRUCTS env var if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { - unsupportedPlpgSqlObjects := fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport) - assessmentReport.UnsupportedPlPgSqlObjects = unsupportedPlpgSqlObjects + assessmentReport.UnsupportedPlPgSqlObjects = fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport) } return nil } @@ -980,31 +982,52 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea var minVersionsFixedIn map[string]*ybversion.YBVersion var minVersionsFixedInSet bool - for _, issue := range schemaAnalysisReport.Issues { - if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, issue.IssueType) { + for _, analyzeIssue := range schemaAnalysisReport.Issues { + if !slices.Contains([]string{UNSUPPORTED_FEATURES_CATEGORY, MIGRATION_CAVEATS_CATEGORY}, analyzeIssue.IssueType) { continue } - issueMatched := lo.Ternary[bool](issueType != "", issueType == issue.Type, strings.Contains(issue.Reason, issueReason)) + issueMatched := lo.Ternary[bool](issueType != "", issueType == analyzeIssue.Type, strings.Contains(analyzeIssue.Reason, issueReason)) if issueMatched { - objectInfo := ObjectInfo{ - ObjectName: issue.ObjectName, - SqlStatement: issue.SqlStatement, - } - link = issue.DocsLink - objects = append(objects, objectInfo) if !minVersionsFixedInSet { - minVersionsFixedIn = issue.MinimumVersionsFixedIn + minVersionsFixedIn = analyzeIssue.MinimumVersionsFixedIn minVersionsFixedInSet = true } - if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { - utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", featureName, minVersionsFixedIn, issue.MinimumVersionsFixedIn) + if !areMinVersionsFixedInEqual(minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) { + utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", featureName, minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) } + + objectInfo := ObjectInfo{ + ObjectName: analyzeIssue.ObjectName, + SqlStatement: analyzeIssue.SqlStatement, + } + link = analyzeIssue.DocsLink + objects = append(objects, objectInfo) + + assessmentReport.AppendIssues(convertAnalyzeSchemaIssueToAssessmentIssue(analyzeIssue, description, minVersionsFixedIn)) } } + return UnsupportedFeature{featureName, objects, displayDDLInHTML, link, description, minVersionsFixedIn} } +// Q: do we no need of displayDDLInHTML in this approach? DDL can always be there for issues in the table if present. +func convertAnalyzeSchemaIssueToAssessmentIssue(analyzeSchemaIssue utils.AnalyzeSchemaIssue, issueDescription string, minVersionsFixedIn map[string]*ybversion.YBVersion) AssessmentIssue { + return AssessmentIssue{ + Category: analyzeSchemaIssue.IssueType, + CategoryDescription: GetCategoryDescription(analyzeSchemaIssue.IssueType), + Type: analyzeSchemaIssue.Type, + Name: analyzeSchemaIssue.Reason, // in convertIssueInstanceToAnalyzeIssue() we assign IssueType to Reason field + Description: issueDescription, // TODO: verify + Impact: analyzeSchemaIssue.Impact, + ObjectType: analyzeSchemaIssue.ObjectType, + ObjectName: analyzeSchemaIssue.ObjectName, + SqlStatement: analyzeSchemaIssue.SqlStatement, + DocsLink: analyzeSchemaIssue.DocsLink, + MinimumVersionFixedIn: minVersionsFixedIn, + } +} + func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for PG...") unsupportedFeatures := make([]UnsupportedFeature, 0) @@ -1115,13 +1138,42 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { unsupportedIndexes = append(unsupportedIndexes, ObjectInfo{ ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), }) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: UNSUPPORTED_FEATURES_CATEGORY, + Type: "", // TODO + Name: UNSUPPORTED_INDEXES_FEATURE, + ObjectType: "INDEX", + ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), + }) } else if objectType == VIRTUAL_COLUMN { virtualColumns = append(virtualColumns, ObjectInfo{ObjectName: objectName}) + assessmentReport.AppendIssues(AssessmentIssue{ + Category: UNSUPPORTED_FEATURES_CATEGORY, + Type: "", // TODO + Name: VIRTUAL_COLUMNS_FEATURE, + ObjectName: objectName, + }) } else if objectType == INHERITED_TYPE { inheritedTypes = append(inheritedTypes, ObjectInfo{ObjectName: objectName}) + assessmentReport.AppendIssues(AssessmentIssue{ + Category: UNSUPPORTED_FEATURES_CATEGORY, + Type: "", // TODO + Name: INHERITED_TYPES_FEATURE, + ObjectName: objectName, + }) } else if objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION { referenceOrTablePartitionPresent = true unsupportedPartitionTypes = append(unsupportedPartitionTypes, ObjectInfo{ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType)}) + + // For oracle migration complexity comes from ora2pg, so defining Impact not required right now + assessmentReport.AppendIssues(AssessmentIssue{ + Category: UNSUPPORTED_FEATURES_CATEGORY, + Type: "", // TODO + Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, + ObjectType: "TABLE", + ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), + }) } } @@ -1139,9 +1191,9 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U if source.DBType != POSTGRESQL { return nil } - analyzeIssues := schemaAnalysisReport.Issues - plpgsqlIssues := lo.Filter(analyzeIssues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { - return issue.IssueType == UNSUPPORTED_PLPGSQL_OBEJCTS + + plpgsqlIssues := lo.Filter(schemaAnalysisReport.Issues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { + return issue.IssueType == UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY }) groupPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.AnalyzeSchemaIssue) string { return issue.Reason @@ -1154,11 +1206,6 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U var minVersionsFixedInSet bool for _, issue := range issues { - objects = append(objects, ObjectInfo{ - ObjectType: issue.ObjectType, - ObjectName: issue.ObjectName, - SqlStatement: issue.SqlStatement, - }) if !minVersionsFixedInSet { minVersionsFixedIn = issue.MinimumVersionsFixedIn minVersionsFixedInSet = true @@ -1166,7 +1213,25 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", reason, minVersionsFixedIn, issue.MinimumVersionsFixedIn) } + + objects = append(objects, ObjectInfo{ + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + }) docsLink = issue.DocsLink + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY, + Type: issue.Type, + Name: reason, + Impact: issue.Impact, // TODO: verify(expected already there since underlying issues are assigned) + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, + }) } feature := UnsupportedFeature{ FeatureName: reason, @@ -1237,11 +1302,21 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error for _, issue := range issues { uqc := utils.UnsupportedQueryConstruct{ Query: issue.SqlStatement, - ConstructTypeName: issue.TypeName, + ConstructTypeName: issue.Name, DocsLink: issue.DocsLink, MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn, } result = append(result, uqc) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY, + Type: issue.Type, + Name: issue.Name, + Impact: issue.Impact, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, + }) } } @@ -1334,6 +1409,27 @@ func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []ut return unsupportedDataTypes, unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB, nil } +func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.TableColumnsDataTypes) []AssessmentIssue { + var assessmentIssues []AssessmentIssue + for _, colInfo := range unsupportedDatatypes { + qualifiedColName := fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName) + issue := AssessmentIssue{ + Category: UNSUPPORTED_DATATYPES_CATEGORY, + CategoryDescription: GetCategoryDescription(UNSUPPORTED_DATATYPES_CATEGORY), + Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" + Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" + Impact: constants.IMPACT_LEVEL_3, + ObjectType: constants.COLUMN, + ObjectName: qualifiedColName, + DocsLink: "", // TODO + MinimumVersionFixedIn: nil, // TODO + } + assessmentIssues = append(assessmentIssues, issue) + } + + return assessmentIssues +} + /* Queries to ignore: - Collected schemas is totally different than source schema list, not containing "" @@ -1427,24 +1523,48 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, - schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_ISSUE)) + schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_DESCRIPTION)) if len(unsupportedDataTypesForLiveMigration) > 0 { columns := make([]ObjectInfo, 0) - for _, col := range unsupportedDataTypesForLiveMigration { - columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) + for _, colInfo := range unsupportedDataTypesForLiveMigration { + columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName, colInfo.DataType)}) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: MIGRATION_CAVEATS_CATEGORY, + CategoryDescription: "", // TODO + Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name + Name: "", // TODO + Impact: constants.IMPACT_LEVEL_1, // Caveat - we don't know the migration is offline/online; + Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, + ObjectType: constants.COLUMN, + ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), + DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, + }) } if len(columns) > 0 { - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, nil}) } } if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { columns := make([]ObjectInfo, 0) - for _, col := range unsupportedDataTypesForLiveMigrationWithFForFB { - columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) + for _, colInfo := range unsupportedDataTypesForLiveMigrationWithFForFB { + columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName, colInfo.DataType)}) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: MIGRATION_CAVEATS_CATEGORY, + CategoryDescription: "", // TODO + Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name + Name: "", // TODO + Impact: constants.IMPACT_LEVEL_1, + Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, + ObjectType: constants.COLUMN, + ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), + DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, + }) } if len(columns) > 0 { - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, nil}) } } migrationCaveats = lo.Filter(migrationCaveats, func(m UnsupportedFeature, _ int) bool { diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 47c5a7c496..2542c14d1b 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1195,13 +1195,15 @@ type AssessmentReport struct { MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` } +// Fields apart from Category, CategoryDescription, TypeName and Impact will be populated only if/when available type AssessmentIssue struct { - Category string + Category string // expected values: feature, query_constrcuts, migration_caveats, plpgsql_objects, datatytpe CategoryDescription string - TypeName string - TypeDescription string - Impact string - ObjectType string + Type string // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS + Name string // Ex: "Stored generated columns are not supported." + Description string + Impact string // Level-1, Level-2, Level-3 (default: Level-1 ??) + ObjectType string // For datatype category, ObjectType will be datatype (for eg "geometry") ObjectName string SqlStatement string DocsLink string @@ -1313,6 +1315,10 @@ func ParseJSONToAssessmentReport(reportPath string) (*AssessmentReport, error) { return &report, nil } +func (ar *AssessmentReport) AppendIssues(issues ...AssessmentIssue) { + ar.Issues = append(ar.Issues, issues...) +} + func (ar *AssessmentReport) GetShardedTablesRecommendation() ([]string, error) { if ar.Sizing == nil { return nil, fmt.Errorf("sizing report is null, can't fetch sharded tables") diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index f4d9cc4fcf..5a2d496be9 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -15,6 +15,11 @@ limitations under the License. */ package cmd +import ( + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + const ( KB = 1024 MB = 1024 * 1024 @@ -102,10 +107,12 @@ const ( REFERENCE_PARTITION = "REFERENCE PARTITION" SYSTEM_PARTITION = "SYSTEM PARTITION" - UNSUPPORTED_FEATURES = "unsupported_features" - UNSUPPORTED_DATATYPES = "unsupported_datatypes" - UNSUPPORTED_PLPGSQL_OBEJCTS = "unsupported_plpgsql_objects" - REPORT_UNSUPPORTED_QUERY_CONSTRUCTS = "REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" + UNSUPPORTED_FEATURES_CATEGORY = "unsupported_features" + UNSUPPORTED_DATATYPES_CATEGORY = "unsupported_datatypes" + UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY = "unsupported_query_constructs" + UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY = "unsupported_plpgsql_objects" + MIGRATION_CAVEATS_CATEGORY = "migration_caveats" + REPORT_UNSUPPORTED_QUERY_CONSTRUCTS = "REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" HTML = "html" JSON = "json" @@ -173,21 +180,14 @@ const ( List of all the features we are reporting as part of Unsupported features and Migration caveats */ const ( - // AssessmentIssue types used in YugabyteD payload - FEATURE = "feature" - DATATYPE = "datatype" - QUERY_CONSTRUCT = "query_construct" // confused: in json for some values we are using space separated and for some snake_case - MIGRATION_CAVEATS = "migration_caveats" - PLPGSQL_OBJECT = "plpgsql_object" - // Description - FEATURE_ISSUE_TYPE_DESCRIPTION = "Features of the source database that are not supported on the target YugabyteDB." - DATATYPE_ISSUE_TYPE_DESCRIPTION = "Data types of the source database that are not supported on the target YugabyteDB." - MIGRATION_CAVEATS_TYPE_DESCRIPTION = "Migration Caveats highlights the current limitations with the migration workflow." - UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION = "Source database queries not supported in YugabyteDB, identified by scanning system tables." - UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PL/pgSQL code block" - SCHEMA_SUMMARY_DESCRIPTION = "Objects that will be created on the target YugabyteDB." - SCHEMA_SUMMARY_DESCRIPTION_ORACLE = SCHEMA_SUMMARY_DESCRIPTION + " Some of the index and sequence names might be different from those in the source database." + FEATURE_CATEGORY_DESCRIPTION = "Features of the source database that are not supported on the target YugabyteDB." + DATATYPE_CATEGORY_DESCRIPTION = "Data types of the source database that are not supported on the target YugabyteDB." + MIGRATION_CAVEATS_CATEGORY_DESCRIPTION = "Migration Caveats highlights the current limitations with the migration workflow." + UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION = "Source database queries not supported in YugabyteDB, identified by scanning system tables." + UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PL/pgSQL code block" + SCHEMA_SUMMARY_DESCRIPTION = "Objects that will be created on the target YugabyteDB." + SCHEMA_SUMMARY_DESCRIPTION_ORACLE = SCHEMA_SUMMARY_DESCRIPTION + " Some of the index and sequence names might be different from those in the source database." //Unsupported Features @@ -222,16 +222,16 @@ const ( // Migration caveats //POSTGRESQL - ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE = "Alter partitioned tables to add Primary Key" - FOREIGN_TABLE_CAVEAT_FEATURE = "Foreign tables" - POLICIES_CAVEAT_FEATURE = "Policies" - UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration" - UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration with Fall-forward/Fallback" - UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE = "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." - UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE = "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." - DESCRIPTION_ADD_PK_TO_PARTITION_TABLE = `After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.` - DESCRIPTION_FOREIGN_TABLES = `During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.` - DESCRIPTION_POLICY_ROLE_ISSUE = `There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.` + ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE = "Alter partitioned tables to add Primary Key" + FOREIGN_TABLE_CAVEAT_FEATURE = "Foreign tables" + POLICIES_CAVEAT_FEATURE = "Policies" + UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration" + UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration with Fall-forward/Fallback" + UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION = "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." + UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION = "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + DESCRIPTION_ADD_PK_TO_PARTITION_TABLE = `After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.` + DESCRIPTION_FOREIGN_TABLES = `During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.` + DESCRIPTION_POLICY_ROLE_DESCRIPTION = `There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.` ) var supportedSourceDBTypes = []string{ORACLE, MYSQL, POSTGRESQL, YUGABYTEDB} @@ -244,3 +244,22 @@ var validSSLModes = map[string][]string{ } var EVENT_BATCH_MAX_RETRY_COUNT = 50 + +// returns the description for a given assessment issue category +func GetCategoryDescription(category string) string { + switch category { + case UNSUPPORTED_FEATURES_CATEGORY, constants.FEATURE: + return FEATURE_CATEGORY_DESCRIPTION + case UNSUPPORTED_DATATYPES_CATEGORY, constants.DATATYPE: + return DATATYPE_CATEGORY_DESCRIPTION + case UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY, constants.QUERY_CONSTRUCT: + return UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION + case UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY, constants.PLPGSQL_OBJECT: + return UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION + case MIGRATION_CAVEATS_CATEGORY: // or constants.MIGRATION_CAVEATS (identical) + return MIGRATION_CAVEATS_CATEGORY_DESCRIPTION + default: + utils.ErrExit("ERROR: unsupported assessment issue category %q", category) + } + return "" +} diff --git a/yb-voyager/src/constants/constants.go b/yb-voyager/src/constants/constants.go index 7912fb1d5c..9e194afe6e 100644 --- a/yb-voyager/src/constants/constants.go +++ b/yb-voyager/src/constants/constants.go @@ -19,10 +19,24 @@ const ( // Database Object types TABLE = "table" FUNCTION = "function" + COLUMN = "column" // Source DB Types YUGABYTEDB = "yugabytedb" POSTGRESQL = "postgresql" ORACLE = "oracle" MYSQL = "mysql" -) + + // AssessmentIssue Categoes - used by YugabyteD payload and Migration Complexity Explainability + // TODO: soon to be renamed as SCHEMA, SCHEMA_PLPGSQL, DML_QUERY, MIGRATION_CAVEAT, "DATATYPE" + FEATURE = "feature" + DATATYPE = "datatype" + QUERY_CONSTRUCT = "query_construct" + MIGRATION_CAVEATS = "migration_caveats" + PLPGSQL_OBJECT = "plpgsql_object" + + // constants for the Impact Buckets + IMPACT_LEVEL_1 = "LEVEL_1" // Represents minimal impact like only the schema ddl + IMPACT_LEVEL_2 = "LEVEL_2" // Represents moderate impact like dml queries which might impact a lot of implementation/assumption in app layer + IMPACT_LEVEL_3 = "LEVEL_3" // Represent significant impact like TABLE INHERITANCE, which doesn't have any simple workaround but can impact multiple objects/apps +) \ No newline at end of file diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index c30d4f8372..47bf663fbf 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -22,8 +22,9 @@ import ( type Issue struct { Type string // (advisory_locks, index_not_supported, etc) - TypeName string // for display - TypeDescription string + Name string // for display + Description string + Impact string Suggestion string GH string DocsLink string @@ -40,3 +41,36 @@ func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { } return v.GreaterThanOrEqual(minVersionFixedInSeries), nil } + +/* + Dynamic Impact Determination (TODO) + - We can define the impact calculator function based on issue type wherever/whenever needed + - Map will have functions only for issue type with dynamic impact determination + + For example: + + type ImpactCalcFunc func(issue QueryIssue, stats *PgStats) string + + var impactCalculators = map[string]ImpactCalcFunc{ + INHERITED_TABLE: inheritedTableImpactCalc, + // etc... + } + + // Example dynamic function + func inheritedTableImpactCalc(i QueryIssue, stats *PgStats) string { + usage := stats.GetUsage(i.ObjectName) // e.g. how many reads/writes + if usage.WritesPerDay > 1000 { + return "LEVEL_2" + } + return "LEVEL_3" + } + + func (i Issue) GetImpact(stats *PgStats) string { + if calc, ok := impactCalculators[i.Type]; ok { + return calc(i, stats) + } + + return lo.Ternary(i.Impact != "", i.Impact, constants.IMPACT_LEVEL_1) + } + +*/ diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 2930b5df4f..7a06bcaa0e 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -20,13 +20,15 @@ import ( "fmt" "strings" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) var generatedColumnsIssue = issue.Issue{ Type: STORED_GENERATED_COLUMNS, - TypeName: "Stored generated columns are not supported.", + Name: "Stored generated columns are not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/10695", Suggestion: "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", @@ -34,13 +36,14 @@ var generatedColumnsIssue = issue.Issue{ func NewGeneratedColumnsIssue(objectType string, objectName string, sqlStatement string, generatedColumns []string) QueryIssue { issue := generatedColumnsIssue - issue.TypeName = issue.TypeName + fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")) + issue.Name = issue.Name + fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unloggedTableIssue = issue.Issue{ Type: UNLOGGED_TABLE, - TypeName: "UNLOGGED tables are not supported yet.", + Name: "UNLOGGED tables are not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/1129/", Suggestion: "Remove UNLOGGED keyword to make it work", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", @@ -56,20 +59,22 @@ func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement st var unsupportedIndexMethodIssue = issue.Issue{ Type: UNSUPPORTED_INDEX_METHOD, - TypeName: "Schema contains %s index which is not supported.", + Name: "Schema contains %s index which is not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1337", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", } func NewUnsupportedIndexMethodIssue(objectType string, objectName string, sqlStatement string, indexAccessMethod string) QueryIssue { issue := unsupportedIndexMethodIssue - issue.TypeName = fmt.Sprintf(unsupportedIndexMethodIssue.TypeName, strings.ToUpper(indexAccessMethod)) + issue.Name = fmt.Sprintf(unsupportedIndexMethodIssue.Name, strings.ToUpper(indexAccessMethod)) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var storageParameterIssue = issue.Issue{ Type: STORAGE_PARAMETER, - TypeName: "Storage parameters are not supported yet.", + Name: "Storage parameters are not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/23467", Suggestion: "Remove the storage parameters from the DDL", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", @@ -82,7 +87,8 @@ func NewStorageParameterIssue(objectType string, objectName string, sqlStatement var setColumnAttributeIssue = issue.Issue{ Type: ALTER_TABLE_SET_COLUMN_ATTRIBUTE, - TypeName: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", + Name: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -95,7 +101,8 @@ func NewSetColumnAttributeIssue(objectType string, objectName string, sqlStateme var alterTableClusterOnIssue = issue.Issue{ Type: ALTER_TABLE_CLUSTER_ON, - TypeName: "ALTER TABLE CLUSTER not supported yet.", + Name: "ALTER TABLE CLUSTER not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -108,7 +115,8 @@ func NewClusterONIssue(objectType string, objectName string, sqlStatement string var alterTableDisableRuleIssue = issue.Issue{ Type: ALTER_TABLE_DISABLE_RULE, - TypeName: "ALTER TABLE name DISABLE RULE not supported yet", + Name: "ALTER TABLE name DISABLE RULE not supported yet", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove this and the rule '%s' from the exported schema to be not enabled on the table.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -123,7 +131,8 @@ func NewAlterTableDisableRuleIssue(objectType string, objectName string, sqlStat var exclusionConstraintIssue = issue.Issue{ Type: EXCLUSION_CONSTRAINTS, - TypeName: "Exclusion constraint is not supported yet", + Name: "Exclusion constraint is not supported yet", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/3944", Suggestion: "Refer docs link for details on possible workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", @@ -138,7 +147,8 @@ func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatem var deferrableConstraintIssue = issue.Issue{ Type: DEFERRABLE_CONSTRAINTS, - TypeName: "DEFERRABLE constraints not supported yet", + Name: "DEFERRABLE constraints not supported yet", + Impact: constants.IMPACT_LEVEL_3, GH: "https://github.com/yugabyte/yugabyte-db/issues/1709", Suggestion: "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", @@ -153,7 +163,8 @@ func NewDeferrableConstraintIssue(objectType string, objectName string, sqlState var multiColumnGinIndexIssue = issue.Issue{ Type: MULTI_COLUMN_GIN_INDEX, - TypeName: "Schema contains gin index on multi column which is not supported.", + Name: "Schema contains gin index on multi column which is not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/10652", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gin-indexes-on-multiple-columns-are-not-supported", } @@ -164,7 +175,8 @@ func NewMultiColumnGinIndexIssue(objectType string, objectName string, sqlStatem var orderedGinIndexIssue = issue.Issue{ Type: ORDERED_GIN_INDEX, - TypeName: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", + Name: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/10653", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#issue-in-some-unsupported-cases-of-gin-indexes", } @@ -175,7 +187,8 @@ func NewOrderedGinIndexIssue(objectType string, objectName string, sqlStatement var policyRoleIssue = issue.Issue{ Type: POLICY_WITH_ROLES, - TypeName: "Policy require roles to be created.", + Name: "Policy require roles to be created.", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", GH: "https://github.com/yugabyte/yb-voyager/issues/1655", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", @@ -183,13 +196,14 @@ var policyRoleIssue = issue.Issue{ func NewPolicyRoleIssue(objectType string, objectName string, sqlStatement string, roles []string) QueryIssue { issue := policyRoleIssue - issue.TypeName = fmt.Sprintf("%s Users - (%s)", issue.TypeName, strings.Join(roles, ",")) + issue.Name = fmt.Sprintf("%s Users - (%s)", issue.Name, strings.Join(roles, ",")) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var constraintTriggerIssue = issue.Issue{ Type: CONSTRAINT_TRIGGER, - TypeName: "CONSTRAINT TRIGGER not supported yet.", + Name: "CONSTRAINT TRIGGER not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1709", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", } @@ -201,7 +215,8 @@ func NewConstraintTriggerIssue(objectType string, objectName string, sqlStatemen var referencingClauseInTriggerIssue = issue.Issue{ Type: REFERENCING_CLAUSE_IN_TRIGGER, - TypeName: "REFERENCING clause (transition tables) not supported yet.", + Name: "REFERENCING clause (transition tables) not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1668", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", } @@ -212,7 +227,8 @@ func NewReferencingClauseTrigIssue(objectType string, objectName string, sqlStat var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ Type: BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, - TypeName: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", + Name: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", + Impact: constants.IMPACT_LEVEL_1, DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", GH: "https://github.com/yugabyte/yugabyte-db/issues/24830", Suggestion: "Create the triggers on individual partitions.", @@ -224,7 +240,8 @@ func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, sql var alterTableAddPKOnPartitionIssue = issue.Issue{ Type: ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, - TypeName: "Adding primary key to a partitioned table is not supported yet.", + Name: "Adding primary key to a partitioned table is not supported yet.", + Impact: constants.IMPACT_LEVEL_1, DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", GH: "https://github.com/yugabyte/yugabyte-db/issues/10074", MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ @@ -241,7 +258,8 @@ func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, sql var expressionPartitionIssue = issue.Issue{ Type: EXPRESSION_PARTITION_WITH_PK_UK, - TypeName: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", + Name: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Remove the Constriant from the table definition", GH: "https://github.com/yugabyte/yb-voyager/issues/698", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", @@ -253,7 +271,8 @@ func NewExpressionPartitionIssue(objectType string, objectName string, sqlStatem var multiColumnListPartition = issue.Issue{ Type: MULTI_COLUMN_LIST_PARTITION, - TypeName: `cannot use "list" partition strategy with more than one column`, + Name: `cannot use "list" partition strategy with more than one column`, + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Make it a single column partition by list or choose other supported Partitioning methods", GH: "https://github.com/yugabyte/yb-voyager/issues/699", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", @@ -265,7 +284,8 @@ func NewMultiColumnListPartition(objectType string, objectName string, sqlStatem var insufficientColumnsInPKForPartition = issue.Issue{ Type: INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, - TypeName: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", + Name: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Add all Partition columns to Primary Key", GH: "https://github.com/yugabyte/yb-voyager/issues/578", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", @@ -273,13 +293,14 @@ var insufficientColumnsInPKForPartition = issue.Issue{ func NewInsufficientColumnInPKForPartition(objectType string, objectName string, sqlStatement string, partitionColumnsNotInPK []string) QueryIssue { issue := insufficientColumnsInPKForPartition - issue.TypeName = fmt.Sprintf("%s - (%s)", issue.TypeName, strings.Join(partitionColumnsNotInPK, ", ")) + issue.Name = fmt.Sprintf("%s - (%s)", issue.Name, strings.Join(partitionColumnsNotInPK, ", ")) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xmlDatatypeIssue = issue.Issue{ Type: XML_DATATYPE, - TypeName: "Unsupported datatype - xml", + Name: "Unsupported datatype - xml", + Impact: constants.IMPACT_LEVEL_3, Suggestion: "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", GH: "https://github.com/yugabyte/yugabyte-db/issues/1043", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", @@ -287,13 +308,14 @@ var xmlDatatypeIssue = issue.Issue{ func NewXMLDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xmlDatatypeIssue - issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + issue.Name = fmt.Sprintf("%s on column - %s", issue.Name, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xidDatatypeIssue = issue.Issue{ Type: XID_DATATYPE, - TypeName: "Unsupported datatype - xid", + Name: "Unsupported datatype - xid", + Impact: constants.IMPACT_LEVEL_3, Suggestion: "Functions for this type e.g. txid_current are not supported in YugabyteDB yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/15638", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", @@ -301,65 +323,70 @@ var xidDatatypeIssue = issue.Issue{ func NewXIDDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xidDatatypeIssue - issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + issue.Name = fmt.Sprintf("%s on column - %s", issue.Name, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var postgisDatatypeIssue = issue.Issue{ Type: POSTGIS_DATATYPES, - TypeName: "Unsupported datatype", + Name: "Unsupported datatype", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/11323", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } func NewPostGisDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := postgisDatatypeIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES, - TypeName: "Unsupported datatype", + Name: "Unsupported datatype", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } func NewUnsupportedDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION, - TypeName: "Unsupported datatype for Live migration", + Name: "Unsupported datatype for Live migration", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB, - TypeName: "Unsupported datatype for Live migration with fall-forward/fallback", + Name: "Unsupported datatype for Live migration with fall-forward/fallback", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationWithFFOrFBIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ Type: PK_UK_ON_COMPLEX_DATATYPE, - TypeName: "Primary key and Unique constraint on column '%s' not yet supported", + Name: "Primary key and Unique constraint on column '%s' not yet supported", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, @@ -370,13 +397,14 @@ func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objec CONSTRAINT_NAME: constraintName, } issue := primaryOrUniqueOnUnsupportedIndexTypesIssue - issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) + issue.Name = fmt.Sprintf(issue.Name, typeName) return newQueryIssue(issue, objectType, objectName, sqlStatement, details) } var indexOnComplexDatatypesIssue = issue.Issue{ Type: INDEX_ON_COMPLEX_DATATYPE, - TypeName: "INDEX on column '%s' not yet supported", + Name: "INDEX on column '%s' not yet supported", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", @@ -384,13 +412,14 @@ var indexOnComplexDatatypesIssue = issue.Issue{ func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string) QueryIssue { issue := indexOnComplexDatatypesIssue - issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) + issue.Name = fmt.Sprintf(issue.Name, typeName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var foreignTableIssue = issue.Issue{ Type: FOREIGN_TABLE, - TypeName: "Foreign tables require manual intervention.", + Name: "Foreign tables require manual intervention.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1627", Suggestion: "SERVER '%s', and USER MAPPING should be created manually on the target to create and use the foreign table", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", @@ -404,7 +433,8 @@ func NewForeignTableIssue(objectType string, objectName string, sqlStatement str var inheritanceIssue = issue.Issue{ Type: INHERITANCE, - TypeName: "TABLE INHERITANCE not supported in YugabyteDB", + Name: "TABLE INHERITANCE not supported in YugabyteDB", + Impact: constants.IMPACT_LEVEL_3, GH: "https://github.com/YugaByte/yugabyte-db/issues/1129", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", } @@ -414,12 +444,13 @@ func NewInheritanceIssue(objectType string, objectName string, sqlStatement stri } var percentTypeSyntax = issue.Issue{ - Type: REFERENCED_TYPE_DECLARATION, - TypeName: "Referenced type declaration of variables", - TypeDescription: "", - Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", - GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + Type: REFERENCED_TYPE_DECLARATION, + Name: "Referenced type declaration of variables", + Impact: constants.IMPACT_LEVEL_1, + Description: "", + Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", + GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", } func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -428,7 +459,8 @@ func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatemen var loDatatypeIssue = issue.Issue{ Type: LARGE_OBJECT_DATATYPE, - TypeName: "Unsupported datatype - lo", + Name: "Unsupported datatype - lo", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", DocsLink: "", // TODO @@ -436,13 +468,14 @@ var loDatatypeIssue = issue.Issue{ func NewLODatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { issue := loDatatypeIssue - issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + issue.Name = fmt.Sprintf("%s on column - %s", issue.Name, colName) return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } var multiRangeDatatypeIssue = issue.Issue{ Type: MULTI_RANGE_DATATYPE, - TypeName: "Unsupported datatype", + Name: "Unsupported datatype", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", GH: "", //TODO DocsLink: "", //TODO @@ -450,13 +483,14 @@ var multiRangeDatatypeIssue = issue.Issue{ func NewMultiRangeDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := multiRangeDatatypeIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var securityInvokerViewIssue = issue.Issue{ Type: SECURITY_INVOKER_VIEWS, - TypeName: "Security Invoker Views not supported yet", + Name: "Security Invoker Views not supported yet", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Security Invoker Views are not yet supported in YugabyteDB, no workaround available currently", GH: "", // TODO DocsLink: "", // TODO @@ -468,7 +502,8 @@ func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatem var foreignKeyReferencesPartitionedTableIssue = issue.Issue{ Type: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, - TypeName: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, + Name: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, + Impact: constants.IMPACT_LEVEL_1, Suggestion: "No workaround available ", GH: "", // TODO DocsLink: "", // TODO diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index e9b8f25923..393841a79e 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -19,16 +19,18 @@ package queryissue import ( "sort" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" ) var advisoryLocksIssue = issue.Issue{ - Type: ADVISORY_LOCKS, - TypeName: "Advisory Locks", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + Type: ADVISORY_LOCKS, + Name: "Advisory Locks", + Impact: constants.IMPACT_LEVEL_2, + Description: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", } func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -36,12 +38,13 @@ func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement st } var systemColumnsIssue = issue.Issue{ - Type: SYSTEM_COLUMNS, - TypeName: "System Columns", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + Type: SYSTEM_COLUMNS, + Name: "System Columns", + Impact: constants.IMPACT_LEVEL_2, + Description: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", } func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -49,12 +52,13 @@ func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement st } var xmlFunctionsIssue = issue.Issue{ - Type: XML_FUNCTIONS, - TypeName: "XML Functions", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + Type: XML_FUNCTIONS, + Name: "XML Functions", + Impact: constants.IMPACT_LEVEL_2, + Description: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", } func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -62,12 +66,13 @@ func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement str } var regexFunctionsIssue = issue.Issue{ - Type: REGEX_FUNCTIONS, - TypeName: "Regex Functions", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: REGEX_FUNCTIONS, + Name: "Regex Functions", + Impact: constants.IMPACT_LEVEL_2, + Description: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -75,12 +80,13 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s } var aggregateFunctionIssue = issue.Issue{ - Type: AGGREGATE_FUNCTION, - TypeName: AGGREGATION_FUNCTIONS_NAME, - TypeDescription: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", + Type: AGGREGATE_FUNCTION, + Name: AGGREGATION_FUNCTIONS_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", } func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -92,12 +98,13 @@ func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatem } var jsonConstructorFunctionsIssue = issue.Issue{ - Type: JSON_CONSTRUCTOR_FUNCTION, - TypeName: JSON_CONSTRUCTOR_FUNCTION_NAME, - TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", + Type: JSON_CONSTRUCTOR_FUNCTION, + Name: JSON_CONSTRUCTOR_FUNCTION_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", } func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -109,12 +116,13 @@ func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlSt } var jsonQueryFunctionIssue = issue.Issue{ - Type: JSON_QUERY_FUNCTION, - TypeName: JSON_QUERY_FUNCTIONS_NAME, - TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", + Type: JSON_QUERY_FUNCTION, + Name: JSON_QUERY_FUNCTIONS_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", } func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -126,12 +134,13 @@ func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatemen } var loFunctionsIssue = issue.Issue{ - Type: LARGE_OBJECT_FUNCTIONS, - TypeName: LARGE_OBJECT_FUNCTIONS_NAME, - TypeDescription: "Large Objects functions are not supported in YugabyteDB", - Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", - GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", - DocsLink: "", //TODO + Type: LARGE_OBJECT_FUNCTIONS, + Name: LARGE_OBJECT_FUNCTIONS_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "Large Objects functions are not supported in YugabyteDB", + Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", + DocsLink: "", //TODO } func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -143,12 +152,13 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin } var jsonbSubscriptingIssue = issue.Issue{ - Type: JSONB_SUBSCRIPTING, - TypeName: JSONB_SUBSCRIPTING_NAME, - TypeDescription: "Jsonb subscripting is not supported in YugabyteDB yet", - Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", - GH: "", - DocsLink: "", //TODO + Type: JSONB_SUBSCRIPTING, + Name: JSONB_SUBSCRIPTING_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "Jsonb subscripting is not supported in YugabyteDB yet", + Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", + GH: "", + DocsLink: "", //TODO } func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -156,12 +166,13 @@ func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatemen } var jsonPredicateIssue = issue.Issue{ - Type: JSON_TYPE_PREDICATE, - TypeName: JSON_TYPE_PREDICATE_NAME, - TypeDescription: "IS JSON predicate expressions not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", //TODO + Type: JSON_TYPE_PREDICATE, + Name: JSON_TYPE_PREDICATE_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "IS JSON predicate expressions not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", //TODO } func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -169,12 +180,13 @@ func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement st } var copyFromWhereIssue = issue.Issue{ - Type: COPY_FROM_WHERE, - TypeName: "COPY FROM ... WHERE", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: COPY_FROM_WHERE, + Name: "COPY FROM ... WHERE", + Impact: constants.IMPACT_LEVEL_2, + Description: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -182,12 +194,13 @@ func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement st } var copyOnErrorIssue = issue.Issue{ - Type: COPY_ON_ERROR, - TypeName: "COPY ... ON_ERROR", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: COPY_ON_ERROR, + Name: "COPY ... ON_ERROR", + Impact: constants.IMPACT_LEVEL_2, + Description: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -195,12 +208,13 @@ func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement stri } var fetchWithTiesIssue = issue.Issue{ - Type: FETCH_WITH_TIES, - TypeName: "FETCH .. WITH TIES", - TypeDescription: "FETCH .. WITH TIES is not supported in YugabyteDB", - Suggestion: "No workaround available right now", - GH: "", - DocsLink: "", //TODO + Type: FETCH_WITH_TIES, + Name: "FETCH .. WITH TIES", + Impact: constants.IMPACT_LEVEL_2, + Description: "FETCH .. WITH TIES is not supported in YugabyteDB", + Suggestion: "No workaround available right now", + GH: "", + DocsLink: "", //TODO } func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { diff --git a/yb-voyager/src/query/queryparser/object_collector.go b/yb-voyager/src/query/queryparser/object_collector.go index ba7561a1b5..9a6020699b 100644 --- a/yb-voyager/src/query/queryparser/object_collector.go +++ b/yb-voyager/src/query/queryparser/object_collector.go @@ -93,7 +93,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { relName := GetStringField(relationMsg, "relname") objectName := utils.BuildObjectName(schemaName, relName) log.Debugf("[IUD] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) - if c.predicate(schemaName, relName, "table") { + if c.predicate(schemaName, relName, constants.TABLE) { c.addObject(objectName) } } diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index b42c547d98..16a8725adf 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -107,6 +107,7 @@ type AnalyzeSchemaIssue struct { ObjectName string `json:"ObjectName"` Reason string `json:"Reason"` Type string `json:"-" xml:"-"` // identifier for issue type ADVISORY_LOCKS, SYSTEM_COLUMNS, etc + Impact string `json:"-" xml:"-"` // temporary field; since currently we generate assessment issue from analyze issue SqlStatement string `json:"SqlStatement,omitempty"` FilePath string `json:"FilePath"` Suggestion string `json:"Suggestion"` From a6b5e4c30d8fb49cfe900350cff27ea8d9eae7cd Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 8 Jan 2025 23:46:22 +0530 Subject: [PATCH 094/105] Implement Migration complexity using the assessment issues (#2161) - now the new algorithm for mig complexity considers all kind of issues including schema, query constructs, plpgsql, datatypes etc... * Removed migration complexity info from analyze schema reports * Fixed the expected mig complexity for pg_assessment_report_test_uqc from LOW to MEDIUM - since the unsupported query constructs are being considered now in mig complexity calculatin * Fixed expected migration complexity for omnibus and osm schema - complexity from Medium->High, and Low->Medium, due to new things - unsupported_datatype(level-3) and change in threshold for HIGH --- .../expected_schema_analysis_report.json | 1 - .../expectedAssessmentReport.json | 2 +- .../expected_schema_analysis_report.json | 1 - .../expectedAssessmentReport.json | 2 +- .../expected_schema_analysis_report.json | 1 - .../expectedAssessmentReport.json | 2 +- .../expected_schema_analysis_report.json | 1 - .../expected_schema_analysis_report.json | 1 - .../expected_schema_analysis_report.json | 1 - .../expected_schema_analysis_report.json | 1 - .../expected_schema_analysis_report.json | 1 - .../expected_schema_analysis_report.json | 1 - yb-voyager/cmd/analyzeSchema.go | 1 - yb-voyager/cmd/assessMigrationCommand.go | 4 +- yb-voyager/cmd/common.go | 125 ----------- yb-voyager/cmd/constants.go | 3 - yb-voyager/cmd/migration_complexity.go | 202 ++++++++++++++++++ yb-voyager/cmd/migration_complexity_test.go | 134 ++++++++++++ .../cmd/templates/schema_analysis_report.html | 5 - .../cmd/templates/schema_analysis_report.txt | 4 - yb-voyager/src/constants/constants.go | 7 +- yb-voyager/src/utils/commonVariables.go | 9 +- 22 files changed, 352 insertions(+), 157 deletions(-) create mode 100644 yb-voyager/cmd/migration_complexity.go create mode 100644 yb-voyager/cmd/migration_complexity_test.go diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index c58df24345..40564ee43a 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "HIGH", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "adventureworks", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index b15b108395..f9f859b2f0 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -1,7 +1,7 @@ { "VoyagerVersion": "IGNORED", "TargetDBVersion": "IGNORED", - "MigrationComplexity": "LOW", + "MigrationComplexity": "MEDIUM", "SchemaSummary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "pg_assessment_report_uqc", diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index bc5c2eee99..c5b66ad796 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "MEDIUM", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_mgd", diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 8cd48f94b6..6bc3e3653b 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -1,6 +1,6 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "MEDIUM", + "MigrationComplexity": "HIGH", "SchemaSummary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_omnibus", diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index a2eccb31e1..bde78722cb 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "MEDIUM", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_omnibus", diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 3aaf994a31..45fb806f22 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -1,6 +1,6 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "LOW", + "MigrationComplexity": "MEDIUM", "SchemaSummary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_osm", diff --git a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json index d4c7d50407..8e78f4cbb8 100755 --- a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "LOW", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_osm", diff --git a/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json index f907e1d6b2..f886d71f44 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "LOW", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_pgtbrus", diff --git a/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json index b697b742cb..33fbca2cce 100644 --- a/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "HIGH", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_rna", diff --git a/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json index c6e31b194d..b2f01e9109 100755 --- a/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "HIGH", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "pg_sakila", diff --git a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json index 9e336fc615..b1f90fab0d 100755 --- a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "LOW", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_is", diff --git a/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json index f40bc041d6..ca3d4ac3ce 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json @@ -1,6 +1,5 @@ { "VoyagerVersion": "IGNORED", - "MigrationComplexity": "MEDIUM", "Summary": { "Description": "Objects that will be created on the target YugabyteDB.", "DbName": "test_stackex", diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 3eb78dc3b5..a002df9efa 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1091,7 +1091,6 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source, detectIssues bool) utils. schemaAnalysisReport.SchemaSummary = reportSchemaSummary(sourceDBConf) schemaAnalysisReport.VoyagerVersion = utils.YB_VOYAGER_VERSION schemaAnalysisReport.TargetDBVersion = targetDbVersion - schemaAnalysisReport.MigrationComplexity = getMigrationComplexity(sourceDBConf.DBType, schemaDir, schemaAnalysisReport) return schemaAnalysisReport } diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index a80de6b4e9..927aba2b6e 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -896,6 +896,9 @@ func generateAssessmentReport() (err error) { addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) + // calculating migration complexity after collecting all assessment issues + assessmentReport.MigrationComplexity = calculateMigrationComplexity(source.DBType, schemaDir, assessmentReport) + assessmentReport.Sizing = migassessment.SizingReport assessmentReport.TableIndexStats, err = assessmentDB.FetchAllStats() if err != nil { @@ -926,7 +929,6 @@ func getAssessmentReportContentFromAnalyzeSchema() error { But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). */ schemaAnalysisReport := analyzeSchemaInternal(&source, true) - assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary assessmentReport.SchemaSummary.Description = lo.Ternary(source.DBType == ORACLE, SCHEMA_SUMMARY_DESCRIPTION_ORACLE, SCHEMA_SUMMARY_DESCRIPTION) diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 2542c14d1b..cc55f737a1 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -16,7 +16,6 @@ limitations under the License. package cmd import ( - "encoding/csv" "encoding/json" "errors" "fmt" @@ -1048,130 +1047,6 @@ func storeTableListInMSR(tableList []sqlname.NameTuple) error { return nil } -var ( - UNSUPPORTED_DATATYPE_XML_ISSUE = fmt.Sprintf("%s - xml", UNSUPPORTED_DATATYPE) - UNSUPPORTED_DATATYPE_XID_ISSUE = fmt.Sprintf("%s - xid", UNSUPPORTED_DATATYPE) - APP_CHANGES_HIGH_THRESHOLD = 5 - APP_CHANGES_MEDIUM_THRESHOLD = 1 - SCHEMA_CHANGES_HIGH_THRESHOLD = math.MaxInt32 - SCHEMA_CHANGES_MEDIUM_THRESHOLD = 20 -) - -var appChanges = []string{ - INHERITANCE_ISSUE_REASON, - CONVERSION_ISSUE_REASON, - DEFERRABLE_CONSTRAINT_ISSUE, - UNSUPPORTED_DATATYPE_XML_ISSUE, - UNSUPPORTED_DATATYPE_XID_ISSUE, - UNSUPPORTED_EXTENSION_ISSUE, // will confirm this -} - -func readEnvForAppOrSchemaCounts() { - APP_CHANGES_HIGH_THRESHOLD = utils.GetEnvAsInt("APP_CHANGES_HIGH_THRESHOLD", APP_CHANGES_HIGH_THRESHOLD) - APP_CHANGES_MEDIUM_THRESHOLD = utils.GetEnvAsInt("APP_CHANGES_MEDIUM_THRESHOLD", APP_CHANGES_MEDIUM_THRESHOLD) - SCHEMA_CHANGES_HIGH_THRESHOLD = utils.GetEnvAsInt("SCHEMA_CHANGES_HIGH_THRESHOLD", SCHEMA_CHANGES_HIGH_THRESHOLD) - SCHEMA_CHANGES_MEDIUM_THRESHOLD = utils.GetEnvAsInt("SCHEMA_CHANGES_MEDIUM_THRESHOLD", SCHEMA_CHANGES_MEDIUM_THRESHOLD) -} - -// Migration complexity calculation from the conversion issues -func getMigrationComplexity(sourceDBType string, schemaDirectory string, analysisReport utils.SchemaReport) string { - if analysisReport.MigrationComplexity != "" { - return analysisReport.MigrationComplexity - } - - if sourceDBType == ORACLE { - mc, err := getMigrationComplexityForOracle(schemaDirectory) - if err != nil { - log.Errorf("failed to get migration complexity for oracle: %v", err) - return "NOT AVAILABLE" - } - return mc - } else if sourceDBType != POSTGRESQL { - return "NOT AVAILABLE" - } - - log.Infof("Calculating migration complexity..") - readEnvForAppOrSchemaCounts() - appChangesCount := 0 - for _, issue := range schemaAnalysisReport.Issues { - for _, appChange := range appChanges { - if strings.Contains(issue.Reason, appChange) { - appChangesCount++ - } - } - } - schemaChangesCount := len(schemaAnalysisReport.Issues) - appChangesCount - - if appChangesCount > APP_CHANGES_HIGH_THRESHOLD || schemaChangesCount > SCHEMA_CHANGES_HIGH_THRESHOLD { - return HIGH - } else if appChangesCount > APP_CHANGES_MEDIUM_THRESHOLD || schemaChangesCount > SCHEMA_CHANGES_MEDIUM_THRESHOLD { - return MEDIUM - } - //LOW in case appChanges == 0 or schemaChanges [0-20] - return LOW -} - -// This is a temporary logic to get migration complexity for oracle based on the migration level from ora2pg report. -// Ideally, we should ALSO be considering the schema analysis report to get the migration complexity. -func getMigrationComplexityForOracle(schemaDirectory string) (string, error) { - ora2pgReportPath := filepath.Join(schemaDirectory, "ora2pg_report.csv") - if !utils.FileOrFolderExists(ora2pgReportPath) { - return "", fmt.Errorf("ora2pg report file not found at %s", ora2pgReportPath) - } - file, err := os.Open(ora2pgReportPath) - if err != nil { - return "", fmt.Errorf("failed to read file %s: %w", ora2pgReportPath, err) - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Error while closing file %s: %v", ora2pgReportPath, err) - } - }() - // Sample file contents - - // "dbi:Oracle:(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = xyz)(PORT = 1521))(CONNECT_DATA = (SERVICE_NAME = DMS)))"; - // "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0";"ASSESS_MIGRATION";"261.62 MB";"1 person-day(s)";"A-2"; - // "0/0/0.00";"0/0/0";"0/0/0";"25/0/6.50";"0/0/0.00";"0/0/0";"0/0/0";"0/0/0";"0/0/0";"3/0/1.00";"3/0/1.00"; - // "44/0/4.90";"27/0/2.70";"9/0/1.80";"4/0/16.00";"5/0/3.00";"2/0/2.00";"125/0/58.90" - // - // X/Y/Z - total/invalid/cost for each type of objects(table,function,etc). Last data element is the sum total. - // total cost = 58.90 units (1 unit = 5 minutes). Therefore total cost is approx 1 person-days. - // column 6 is Migration level. - // Migration levels: - // A - Migration that might be run automatically - // B - Migration with code rewrite and a human-days cost up to 5 days - // C - Migration with code rewrite and a human-days cost above 5 days - // Technical levels: - // 1 = trivial: no stored functions and no triggers - // 2 = easy: no stored functions but with triggers, no manual rewriting - // 3 = simple: stored functions and/or triggers, no manual rewriting - // 4 = manual: no stored functions but with triggers or views with code rewriting - // 5 = difficult: stored functions and/or triggers with code rewriting - reader := csv.NewReader(file) - reader.Comma = ';' - rows, err := reader.ReadAll() - if err != nil { - log.Errorf("error reading csv file %s: %v", ora2pgReportPath, err) - return "", fmt.Errorf("error reading csv file %s: %w", ora2pgReportPath, err) - } - if len(rows) > 1 { - return "", fmt.Errorf("invalid ora2pg report file format. Expected 1 row, found %d. contents = %v", len(rows), rows) - } - reportData := rows[0] - migrationLevel := strings.Split(reportData[5], "-")[0] - - switch migrationLevel { - case "A": - return LOW, nil - case "B": - return MEDIUM, nil - case "C": - return HIGH, nil - default: - return "", fmt.Errorf("invalid migration level [%s] found in ora2pg report %v", migrationLevel, reportData) - } -} - // ===================================================================== // TODO: consider merging all unsupported field with single AssessmentReport struct member as AssessmentIssue diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 5a2d496be9..25ee2ecba8 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -62,9 +62,6 @@ const ( ROW_UPDATE_STATUS_IN_PROGRESS = 1 ROW_UPDATE_STATUS_COMPLETED = 3 COLOCATION_CLAUSE = "colocation" - LOW = "LOW" - MEDIUM = "MEDIUM" - HIGH = "HIGH" //phase names used in call-home payload ANALYZE_PHASE = "analyze-schema" EXPORT_SCHEMA_PHASE = "export-schema" diff --git a/yb-voyager/cmd/migration_complexity.go b/yb-voyager/cmd/migration_complexity.go new file mode 100644 index 0000000000..ae0b379759 --- /dev/null +++ b/yb-voyager/cmd/migration_complexity.go @@ -0,0 +1,202 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "encoding/csv" + "fmt" + "math" + "os" + "path/filepath" + "strings" + + "github.com/samber/lo" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + "golang.org/x/exp/slices" +) + +const NOT_AVAILABLE = "NOT AVAILABLE" + +var ( + LEVEL_1_MEDIUM_THRESHOLD = 20 + LEVEL_1_HIGH_THRESHOLD = math.MaxInt32 + LEVEL_2_MEDIUM_THRESHOLD = 10 + LEVEL_2_HIGH_THRESHOLD = 100 + LEVEL_3_MEDIUM_THRESHOLD = 0 + LEVEL_3_HIGH_THRESHOLD = 4 +) + +// Migration complexity calculation from the conversion issues +func calculateMigrationComplexity(sourceDBType string, schemaDirectory string, assessmentReport AssessmentReport) string { + if sourceDBType != ORACLE && sourceDBType != POSTGRESQL { + return NOT_AVAILABLE + } + + log.Infof("calculating migration complexity for %s...", sourceDBType) + switch sourceDBType { + case ORACLE: + migrationComplexity, err := calculateMigrationComplexityForOracle(schemaDirectory) + if err != nil { + log.Errorf("failed to get migration complexity for oracle: %v", err) + return NOT_AVAILABLE + } + return migrationComplexity + case POSTGRESQL: + return calculateMigrationComplexityForPG(assessmentReport) + default: + panic(fmt.Sprintf("unsupported source db type '%s' for migration complexity", sourceDBType)) + } +} + +func calculateMigrationComplexityForPG(assessmentReport AssessmentReport) string { + counts := lo.CountValuesBy(assessmentReport.Issues, func(issue AssessmentIssue) string { + return issue.Impact + }) + + level1IssueCount := counts[constants.IMPACT_LEVEL_1] + level2IssueCount := counts[constants.IMPACT_LEVEL_2] + level3IssueCount := counts[constants.IMPACT_LEVEL_3] + + utils.PrintAndLog("issue counts: level-1=%d, level-2=%d, level-3=%d\n", level1IssueCount, level2IssueCount, level3IssueCount) + // Determine complexity for each level + comp1 := getComplexityForLevel(constants.IMPACT_LEVEL_1, level1IssueCount) + comp2 := getComplexityForLevel(constants.IMPACT_LEVEL_2, level2IssueCount) + comp3 := getComplexityForLevel(constants.IMPACT_LEVEL_3, level3IssueCount) + complexities := []string{comp1, comp2, comp3} + + // If ANY level is HIGH => final is HIGH + if slices.Contains(complexities, constants.MIGRATION_COMPLEXITY_HIGH) { + return constants.MIGRATION_COMPLEXITY_HIGH + } + // Else if ANY level is MEDIUM => final is MEDIUM + if slices.Contains(complexities, constants.MIGRATION_COMPLEXITY_MEDIUM) { + return constants.MIGRATION_COMPLEXITY_MEDIUM + } + return constants.MIGRATION_COMPLEXITY_LOW +} + +// This is a temporary logic to get migration complexity for oracle based on the migration level from ora2pg report. +// Ideally, we should ALSO be considering the schema analysis report to get the migration complexity. +func calculateMigrationComplexityForOracle(schemaDirectory string) (string, error) { + ora2pgReportPath := filepath.Join(schemaDirectory, "ora2pg_report.csv") + if !utils.FileOrFolderExists(ora2pgReportPath) { + return "", fmt.Errorf("ora2pg report file not found at %s", ora2pgReportPath) + } + file, err := os.Open(ora2pgReportPath) + if err != nil { + return "", fmt.Errorf("failed to read file %s: %w", ora2pgReportPath, err) + } + defer func() { + if err := file.Close(); err != nil { + log.Errorf("Error while closing file %s: %v", ora2pgReportPath, err) + } + }() + // Sample file contents + + // "dbi:Oracle:(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = xyz)(PORT = 1521))(CONNECT_DATA = (SERVICE_NAME = DMS)))"; + // "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0";"ASSESS_MIGRATION";"261.62 MB";"1 person-day(s)";"A-2"; + // "0/0/0.00";"0/0/0";"0/0/0";"25/0/6.50";"0/0/0.00";"0/0/0";"0/0/0";"0/0/0";"0/0/0";"3/0/1.00";"3/0/1.00"; + // "44/0/4.90";"27/0/2.70";"9/0/1.80";"4/0/16.00";"5/0/3.00";"2/0/2.00";"125/0/58.90" + // + // X/Y/Z - total/invalid/cost for each type of objects(table,function,etc). Last data element is the sum total. + // total cost = 58.90 units (1 unit = 5 minutes). Therefore total cost is approx 1 person-days. + // column 6 is Migration level. + // Migration levels: + // A - Migration that might be run automatically + // B - Migration with code rewrite and a human-days cost up to 5 days + // C - Migration with code rewrite and a human-days cost above 5 days + // Technical levels: + // 1 = trivial: no stored functions and no triggers + // 2 = easy: no stored functions but with triggers, no manual rewriting + // 3 = simple: stored functions and/or triggers, no manual rewriting + // 4 = manual: no stored functions but with triggers or views with code rewriting + // 5 = difficult: stored functions and/or triggers with code rewriting + reader := csv.NewReader(file) + reader.Comma = ';' + rows, err := reader.ReadAll() + if err != nil { + log.Errorf("error reading csv file %s: %v", ora2pgReportPath, err) + return "", fmt.Errorf("error reading csv file %s: %w", ora2pgReportPath, err) + } + if len(rows) > 1 { + return "", fmt.Errorf("invalid ora2pg report file format. Expected 1 row, found %d. contents = %v", len(rows), rows) + } + reportData := rows[0] + migrationLevel := strings.Split(reportData[5], "-")[0] + + switch migrationLevel { + case "A": + return constants.MIGRATION_COMPLEXITY_LOW, nil + case "B": + return constants.MIGRATION_COMPLEXITY_MEDIUM, nil + case "C": + return constants.MIGRATION_COMPLEXITY_HIGH, nil + default: + return "", fmt.Errorf("invalid migration level [%s] found in ora2pg report %v", migrationLevel, reportData) + } +} + +// getComplexityLevel returns LOW, MEDIUM, or HIGH for a given impact level & count +func getComplexityForLevel(level string, count int) string { + switch level { + // ------------------------------------------------------- + // LEVEL_1: + // - LOW if count <= 20 + // - MEDIUM if 20 < count < math.MaxInt32 + // - HIGH if count >= math.MaxInt32 (not possible) + // ------------------------------------------------------- + case constants.IMPACT_LEVEL_1: + if count <= LEVEL_1_MEDIUM_THRESHOLD { + return constants.MIGRATION_COMPLEXITY_LOW + } else if count <= LEVEL_1_HIGH_THRESHOLD { + return constants.MIGRATION_COMPLEXITY_MEDIUM + } + return constants.MIGRATION_COMPLEXITY_HIGH + + // ------------------------------------------------------- + // LEVEL_2: + // - LOW if count <= 10 + // - MEDIUM if 10 < count <= 100 + // - HIGH if count > 100 + // ------------------------------------------------------- + case constants.IMPACT_LEVEL_2: + if count <= LEVEL_2_MEDIUM_THRESHOLD { + return constants.MIGRATION_COMPLEXITY_LOW + } else if count <= LEVEL_2_HIGH_THRESHOLD { + return constants.MIGRATION_COMPLEXITY_MEDIUM + } + return constants.MIGRATION_COMPLEXITY_HIGH + + // ------------------------------------------------------- + // LEVEL_3: + // - LOW if count == 0 + // - MEDIUM if 0 < count <= 4 + // - HIGH if count > 4 + // ------------------------------------------------------- + case constants.IMPACT_LEVEL_3: + if count <= LEVEL_3_MEDIUM_THRESHOLD { + return constants.MIGRATION_COMPLEXITY_LOW + } else if count <= LEVEL_3_HIGH_THRESHOLD { + return constants.MIGRATION_COMPLEXITY_MEDIUM + } + return constants.MIGRATION_COMPLEXITY_HIGH + + default: + panic(fmt.Sprintf("unknown impact level %s for determining complexity", level)) + } +} diff --git a/yb-voyager/cmd/migration_complexity_test.go b/yb-voyager/cmd/migration_complexity_test.go new file mode 100644 index 0000000000..7916c5bc04 --- /dev/null +++ b/yb-voyager/cmd/migration_complexity_test.go @@ -0,0 +1,134 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" +) + +func TestGetComplexityForLevel(t *testing.T) { + testCases := []struct { + level string + count int + expected string + desc string + }{ + // ------------------------------- + // LEVEL_1 test cases + // ------------------------------- + { + level: constants.IMPACT_LEVEL_1, + count: 0, + expected: constants.MIGRATION_COMPLEXITY_LOW, + desc: "L1, count=0 => LOW", + }, + { + level: constants.IMPACT_LEVEL_1, + count: 20, + expected: constants.MIGRATION_COMPLEXITY_LOW, + desc: "L1, count=20 => LOW", + }, + { + level: constants.IMPACT_LEVEL_1, + count: 21, + expected: constants.MIGRATION_COMPLEXITY_MEDIUM, + desc: "L1, count=21 => MEDIUM", + }, + { + level: constants.IMPACT_LEVEL_1, + count: 999999999, + expected: constants.MIGRATION_COMPLEXITY_MEDIUM, + desc: "L1, big count => MEDIUM", + }, + + // ------------------------------- + // LEVEL_2 test cases + // ------------------------------- + { + level: constants.IMPACT_LEVEL_2, + count: 0, + expected: constants.MIGRATION_COMPLEXITY_LOW, + desc: "L2, count=0 => LOW", + }, + { + level: constants.IMPACT_LEVEL_2, + count: 10, + expected: constants.MIGRATION_COMPLEXITY_LOW, + desc: "L2, count=10 => LOW", + }, + { + level: constants.IMPACT_LEVEL_2, + count: 11, + expected: constants.MIGRATION_COMPLEXITY_MEDIUM, + desc: "L2, count=11 => MEDIUM", + }, + { + level: constants.IMPACT_LEVEL_2, + count: 100, + expected: constants.MIGRATION_COMPLEXITY_MEDIUM, + desc: "L2, count=100 => MEDIUM", + }, + { + level: constants.IMPACT_LEVEL_2, + count: 101, + expected: constants.MIGRATION_COMPLEXITY_HIGH, + desc: "L2, count=101 => HIGH", + }, + + // ------------------------------- + // LEVEL_3 test cases + // ------------------------------- + { + level: constants.IMPACT_LEVEL_3, + count: 0, + expected: constants.MIGRATION_COMPLEXITY_LOW, + desc: "L3, count=0 => LOW", + }, + { + level: constants.IMPACT_LEVEL_3, + count: 1, + expected: constants.MIGRATION_COMPLEXITY_MEDIUM, + desc: "L3, count=1 => MEDIUM", + }, + { + level: constants.IMPACT_LEVEL_3, + count: 4, + expected: constants.MIGRATION_COMPLEXITY_MEDIUM, + desc: "L3, count=4 => MEDIUM", + }, + { + level: constants.IMPACT_LEVEL_3, + count: 5, + expected: constants.MIGRATION_COMPLEXITY_HIGH, + desc: "L3, count=5 => HIGH", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + actual := getComplexityForLevel(tc.level, tc.count) + assert.Equal(t, tc.expected, actual, + "Level=%s, Count=%d => Expected %s, Got %s", + tc.level, tc.count, tc.expected, actual, + ) + }) + } +} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.html b/yb-voyager/cmd/templates/schema_analysis_report.html index 562ee4f50d..5daff91b06 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.html +++ b/yb-voyager/cmd/templates/schema_analysis_report.html @@ -69,11 +69,6 @@

      Migration Information

     {{end}} - {{if eq .MigrationComplexity "NOT AVAILABLE"}} - - {{else}} - - {{end}}
    ObjectTotal CountValid CountInvalid CountObject NamesDetails
    ObjectTotal ObjectsObjects Without IssuesObjects With IssuesObject NamesDetails
    {{ .FeatureName }}Link + {{ if $supportedVerStr }} + Supported in Versions: {{ $supportedVerStr }}
    + {{ end }} + {{ if $docsLink }} + Docs Link + {{ else }} + N/A + {{ end }} +
    Schema Name{{ join .SchemaSummary.SchemaNames ", " }}
    DB Version{{ .SchemaSummary.DBVersion }}
    Migration Complexity: {{ .MigrationComplexity }}
    diff --git a/yb-voyager/cmd/templates/schema_analysis_report.txt b/yb-voyager/cmd/templates/schema_analysis_report.txt index 2d046227bf..1c02639f03 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.txt +++ b/yb-voyager/cmd/templates/schema_analysis_report.txt @@ -10,10 +10,6 @@ Database Name : {{ .SchemaSummary.DBName }} Schema Name(s) : {{ join .SchemaSummary.SchemaNames ", " }} DB Version : {{ .SchemaSummary.DBVersion }} Target DB Version : {{ .TargetDBVersion }} -{{if eq .MigrationComplexity "NOT AVAILABLE"}} -{{else}} -Migration Complexity : {{ .MigrationComplexity }} -{{end}} Schema Summary diff --git a/yb-voyager/src/constants/constants.go b/yb-voyager/src/constants/constants.go index 9e194afe6e..c9104002ee 100644 --- a/yb-voyager/src/constants/constants.go +++ b/yb-voyager/src/constants/constants.go @@ -39,4 +39,9 @@ const ( IMPACT_LEVEL_1 = "LEVEL_1" // Represents minimal impact like only the schema ddl IMPACT_LEVEL_2 = "LEVEL_2" // Represents moderate impact like dml queries which might impact a lot of implementation/assumption in app layer IMPACT_LEVEL_3 = "LEVEL_3" // Represent significant impact like TABLE INHERITANCE, which doesn't have any simple workaround but can impact multiple objects/apps -) \ No newline at end of file + + // constants for migration complexity + MIGRATION_COMPLEXITY_LOW = "LOW" + MIGRATION_COMPLEXITY_MEDIUM = "MEDIUM" + MIGRATION_COMPLEXITY_HIGH = "HIGH" +) diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index 16a8725adf..38a924486e 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -74,11 +74,10 @@ var WaitChannel = make(chan int) // ================== Schema Report ============================== type SchemaReport struct { - VoyagerVersion string `json:"VoyagerVersion"` - TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` - MigrationComplexity string `json:"MigrationComplexity"` - SchemaSummary SchemaSummary `json:"Summary"` - Issues []AnalyzeSchemaIssue `json:"Issues"` + VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` + SchemaSummary SchemaSummary `json:"Summary"` + Issues []AnalyzeSchemaIssue `json:"Issues"` } type SchemaSummary struct { From ac37160df25e74d523291ae531a71870da21130d Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:31:25 +0530 Subject: [PATCH 095/105] Reporting unique nulls not distinct in analyze schema and assessment report (#2125) --- .../schema/tables/INDEXES_table.sql | 7 +- .../dummy-export-dir/schema/tables/table.sql | 36 ++ .../tests/analyze-schema/expected_issues.json | 40 ++ migtests/tests/analyze-schema/summary.json | 13 +- .../expectedAssessmentReport.json | 346 +++++++++++++++++- .../pg_assessment_report.sql | 42 +++ yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/src/query/queryissue/constants.go | 3 + yb-voyager/src/query/queryissue/detectors.go | 44 +++ yb-voyager/src/query/queryissue/issues_ddl.go | 13 + .../src/query/queryissue/issues_ddl_test.go | 20 + .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 19 +- .../src/query/queryparser/helpers_protomsg.go | 24 ++ .../src/query/queryparser/traversal_proto.go | 1 + 15 files changed, 592 insertions(+), 18 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql index efc811b0f2..3457b1c67d 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql @@ -103,4 +103,9 @@ CREATE INDEX idx_udt1 on test_udt(home_address1); CREATE INDEX idx_enum on test_udt(some_field); -CREATE INDEX "idx&_enum2" on test_udt((some_field::non_public.enum_test)); \ No newline at end of file +CREATE INDEX "idx&_enum2" on test_udt((some_field::non_public.enum_test)); + +-- Create a unique index on a column with NULLs with the NULLS NOT DISTINCT option +CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email + ON users_unique_nulls_not_distinct_index (email) + NULLS NOT DISTINCT; diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 9ea26b115f..48908efa2d 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -429,3 +429,39 @@ CREATE TABLE timestamptz_multirange_table ( global_event_times tstzmultirange ); +-- Testing tables with unique nulls not distinct constraints + +-- Control case +CREATE TABLE users_unique_nulls_distinct ( + id SERIAL PRIMARY KEY, + email TEXT, + UNIQUE (email) +); + +CREATE TABLE users_unique_nulls_not_distinct ( + id SERIAL PRIMARY KEY, + email TEXT, + UNIQUE NULLS NOT DISTINCT (email) +); + +CREATE TABLE sales_unique_nulls_not_distinct ( + store_id INT, + product_id INT, + sale_date DATE, + UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date) +); + +CREATE TABLE sales_unique_nulls_not_distinct_alter ( + store_id INT, + product_id INT, + sale_date DATE +); + +ALTER TABLE sales_unique_nulls_not_distinct_alter + ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date); + +-- Create a unique index on a column with NULLs with the NULLS NOT DISTINCT option +CREATE TABLE users_unique_nulls_not_distinct_index ( + id INTEGER PRIMARY KEY, + email TEXT +); diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 6f5af66fbe..8baa84606e 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2060,5 +2060,45 @@ "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", "GH": "", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "users_unique_nulls_not_distinct", + "Reason": "Unique Nulls Not Distinct", + "SqlStatement": "CREATE TABLE users_unique_nulls_not_distinct (\n id SERIAL PRIMARY KEY,\n email TEXT,\n UNIQUE NULLS NOT DISTINCT (email)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "sales_unique_nulls_not_distinct", + "Reason": "Unique Nulls Not Distinct", + "SqlStatement": "CREATE TABLE sales_unique_nulls_not_distinct (\n store_id INT,\n product_id INT,\n sale_date DATE,\n UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "sales_unique_nulls_not_distinct_alter", + "Reason": "Unique Nulls Not Distinct", + "SqlStatement": "ALTER TABLE sales_unique_nulls_not_distinct_alter\n\tADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "users_unique_nulls_not_distinct_index_email ON users_unique_nulls_not_distinct_index", + "Reason": "Unique Nulls Not Distinct", + "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email\n ON users_unique_nulls_not_distinct_index (email)\n NULLS NOT DISTINCT;", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 8cc96548fa..2052fbd69e 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,15 +26,14 @@ }, { "ObjectType": "TABLE", - "TotalCount": 59, - "InvalidCount": 50, - "ObjectNames": "public.json_data, employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, - + "TotalCount": 64, + "InvalidCount": 53, + "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, public.json_data, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter, users_unique_nulls_not_distinct_index" }, { "ObjectType": "INDEX", - "TotalCount": 43, - "InvalidCount": 39, - "ObjectNames": "idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", + "TotalCount": 44, + "InvalidCount": 40, + "ObjectNames": "film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt, users_unique_nulls_not_distinct_index_email ON users_unique_nulls_not_distinct_index", "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." }, { diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index c341d8b26a..9ded5a5edf 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -44,15 +44,15 @@ }, { "ObjectType": "TABLE", - "TotalCount": 83, - "InvalidCount": 36, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "TotalCount": 93, + "InvalidCount": 42, + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.sales_unique_nulls_not_distinct, public.sales_unique_nulls_not_distinct_alter, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.users_unique_nulls_distinct, public.users_unique_nulls_not_distinct, public.users_unique_nulls_not_distinct_index, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.sales_unique_nulls_not_distinct, schema2.sales_unique_nulls_not_distinct_alter, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.users_unique_nulls_distinct, schema2.users_unique_nulls_not_distinct, schema2.users_unique_nulls_not_distinct_index, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", - "TotalCount": 26, - "InvalidCount": 22, - "ObjectNames": "idx8 ON public.combined_tbl, idx9 ON public.combined_tbl, idx1 ON public.combined_tbl, idx2 ON public.combined_tbl, idx3 ON public.combined_tbl, idx4 ON public.combined_tbl, idx5 ON public.combined_tbl, idx6 ON public.combined_tbl, idx7 ON public.combined_tbl, idx_array ON public.documents, idx_box_data ON public.mixed_data_types_table1, idx_box_data_brin ON public.mixed_data_types_table1, idx_citext ON public.citext_type, idx_citext1 ON public.citext_type, idx_citext2 ON public.citext_type, idx_inet ON public.inet_type, idx_inet1 ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_point_data ON public.mixed_data_types_table1, idx_valid ON public.test_jsonb, tsquery_idx ON public.ts_query_table, tsvector_idx ON public.documents, idx_box_data ON schema2.mixed_data_types_table1, idx_box_data_spgist ON schema2.mixed_data_types_table1, idx_point_data ON schema2.mixed_data_types_table1" + "TotalCount": 28, + "InvalidCount": 24, + "ObjectNames": "idx1 ON public.combined_tbl, idx2 ON public.combined_tbl, idx3 ON public.combined_tbl, idx4 ON public.combined_tbl, idx5 ON public.combined_tbl, idx6 ON public.combined_tbl, idx7 ON public.combined_tbl, idx8 ON public.combined_tbl, idx9 ON public.combined_tbl, idx_array ON public.documents, idx_box_data ON public.mixed_data_types_table1, idx_box_data ON schema2.mixed_data_types_table1, idx_box_data_brin ON public.mixed_data_types_table1, idx_box_data_spgist ON schema2.mixed_data_types_table1, idx_citext ON public.citext_type, idx_citext1 ON public.citext_type, idx_citext2 ON public.citext_type, idx_inet ON public.inet_type, idx_inet1 ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_point_data ON public.mixed_data_types_table1, idx_point_data ON schema2.mixed_data_types_table1, idx_valid ON public.test_jsonb, tsquery_idx ON public.ts_query_table, tsvector_idx ON public.documents, users_unique_nulls_not_distinct_index_email ON public.users_unique_nulls_not_distinct_index, users_unique_nulls_not_distinct_index_email ON schema2.users_unique_nulls_not_distinct_index" }, { "ObjectType": "FUNCTION", @@ -75,7 +75,7 @@ "ObjectType": "VIEW", "TotalCount": 10, "InvalidCount": 6, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view" + "ObjectNames": "public.ordersentry_view, public.sales_employees, public.top_employees_view, public.view_explicit_security_invoker, schema2.sales_employees, schema2.top_employees_view, test_views.v1, test_views.v2, test_views.v3, test_views.v4" }, { "ObjectType": "TRIGGER", @@ -186,9 +186,19 @@ "schema2.numeric_multirange_table", "schema2.timestamp_multirange_table", "schema2.timestamptz_multirange_table", - "public.employeesforview" + "public.employeesforview", + "schema2.users_unique_nulls_distinct", + "schema2.users_unique_nulls_not_distinct", + "schema2.sales_unique_nulls_not_distinct", + "public.users_unique_nulls_not_distinct_index", + "schema2.users_unique_nulls_not_distinct_index", + "schema2.sales_unique_nulls_not_distinct_alter", + "public.users_unique_nulls_distinct", + "public.users_unique_nulls_not_distinct", + "public.sales_unique_nulls_not_distinct", + "public.sales_unique_nulls_not_distinct_alter" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 89 objects (81 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 109 objects (91 tables/materialized views and 18 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -678,10 +688,272 @@ } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Unique Nulls Not Distinct", + "Objects": [ + { + "ObjectName": "users_unique_nulls_not_distinct_index_email ON public.users_unique_nulls_not_distinct_index", + "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email ON public.users_unique_nulls_not_distinct_index USING btree (email) NULLS NOT DISTINCT;" + }, + { + "ObjectName": "users_unique_nulls_not_distinct_index_email ON schema2.users_unique_nulls_not_distinct_index", + "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email ON schema2.users_unique_nulls_not_distinct_index USING btree (email) NULLS NOT DISTINCT;" + }, + { + "ObjectName": "public.sales_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY public.sales_unique_nulls_not_distinct\n ADD CONSTRAINT sales_unique_nulls_not_distin_store_id_product_id_sale_date_key UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "public.sales_unique_nulls_not_distinct_alter", + "SqlStatement": "ALTER TABLE ONLY public.sales_unique_nulls_not_distinct_alter\n ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "public.users_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY public.users_unique_nulls_not_distinct\n ADD CONSTRAINT users_unique_nulls_not_distinct_email_key UNIQUE NULLS NOT DISTINCT (email);" + }, + { + "ObjectName": "schema2.sales_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY schema2.sales_unique_nulls_not_distinct\n ADD CONSTRAINT sales_unique_nulls_not_distin_store_id_product_id_sale_date_key UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "schema2.sales_unique_nulls_not_distinct_alter", + "SqlStatement": "ALTER TABLE ONLY schema2.sales_unique_nulls_not_distinct_alter\n ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "schema2.users_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY schema2.users_unique_nulls_not_distinct\n ADD CONSTRAINT users_unique_nulls_not_distinct_email_key UNIQUE NULLS NOT DISTINCT (email);" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distinct_alter_unique", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.sales_unique_nulls_not_distinct_alter", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distin_store_id_product_id_sale_date_key", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.sales_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.users_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.users_unique_nulls_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distinct_alter_unique", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.sales_unique_nulls_not_distinct_alter", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distin_store_id_product_id_sale_date_key", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.sales_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.users_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.users_unique_nulls_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distinct_alter", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distinct_alter", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "public", "ObjectName": "Recipients", @@ -2319,6 +2591,62 @@ "ObjectType": "", "ParentTableName": null, "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct_index_email", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.users_unique_nulls_not_distinct_index", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct_index", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct_index", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct_index_email", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.users_unique_nulls_not_distinct_index", + "SizeInBytes": 8192 } ], diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index 72e5ea1e74..e8882d3a91 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -430,3 +430,45 @@ WITH (security_invoker = true) AS SELECT employee_id, first_name FROM public.employees; +-- Testing tables with unique nulls not distinct constraints + +-- Control case +CREATE TABLE users_unique_nulls_distinct ( + id INTEGER PRIMARY KEY, + email TEXT, + UNIQUE (email) +); + +CREATE TABLE users_unique_nulls_not_distinct ( + id INTEGER PRIMARY KEY, + email TEXT, + UNIQUE NULLS NOT DISTINCT (email) +); + +CREATE TABLE sales_unique_nulls_not_distinct ( + store_id INT, + product_id INT, + sale_date DATE, + UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date) +); + +CREATE TABLE sales_unique_nulls_not_distinct_alter ( + store_id INT, + product_id INT, + sale_date DATE +); + +ALTER TABLE sales_unique_nulls_not_distinct_alter + ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date); + +-- Create a unique index on a column with NULLs with the NULLS NOT DISTINCT option +CREATE TABLE users_unique_nulls_not_distinct_index ( + id INTEGER PRIMARY KEY, + email TEXT +); + +CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email + ON users_unique_nulls_not_distinct_index (email) + NULLS NOT DISTINCT; + + diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 927aba2b6e..2774b98b44 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1068,6 +1068,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.UNIQUE_NULLS_NOT_DISTINCT_NAME, "", queryissue.UNIQUE_NULLS_NOT_DISTINCT, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index edb8edf27d..2db0cab2e2 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -81,6 +81,9 @@ const ( FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE = "FOREIGN_KEY_REFERENCED_PARTITIONED_TABLE" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME = "Foreign key constraint references partitioned table" + + UNIQUE_NULLS_NOT_DISTINCT = "UNIQUE_NULLS_NOT_DISTINCT" + UNIQUE_NULLS_NOT_DISTINCT_NAME = "Unique Nulls Not Distinct" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 1bc45b0999..1ba0a53362 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -453,6 +453,50 @@ func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { return issues } +type UniqueNullsNotDistinctDetector struct { + query string + detected bool +} + +func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDetector { + return &UniqueNullsNotDistinctDetector{ + query: query, + } +} + +// Detect checks if a unique constraint is defined which has nulls not distinct +func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEXSTMT_NODE { + indexStmt, err := queryparser.ProtoAsIndexStmt(msg) + if err != nil { + return err + } + + if indexStmt.Unique && indexStmt.NullsNotDistinct { + d.detected = true + } + } else if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_CONSTRAINT_NODE { + constraintNode, err := queryparser.ProtoAsTableConstraint(msg) + if err != nil { + return err + } + + if constraintNode.Contype == queryparser.UNIQUE_CONSTR_TYPE && constraintNode.NullsNotDistinct { + d.detected = true + } + } + + return nil +} + +func (d *UniqueNullsNotDistinctDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.detected { + issues = append(issues, NewUniqueNullsNotDistinctIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type JsonPredicateExprDetector struct { query string detected bool diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 7a06bcaa0e..c6eb5f3955 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -515,3 +515,16 @@ func NewForeignKeyReferencesPartitionedTableIssue(objectType string, objectName } return newQueryIssue(foreignKeyReferencesPartitionedTableIssue, objectType, objectName, SqlStatement, details) } + +var uniqueNullsNotDistinctIssue = issue.Issue{ + Type: UNIQUE_NULLS_NOT_DISTINCT, + Name: UNIQUE_NULLS_NOT_DISTINCT_NAME, + Impact: constants.IMPACT_LEVEL_1, + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewUniqueNullsNotDistinctIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(uniqueNullsNotDistinctIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 55642bd8e4..53498dffa8 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -287,6 +287,23 @@ func testForeignKeyReferencesPartitionedTableIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `cannot reference partitioned table "abc1"`, foreignKeyReferencesPartitionedTableIssue) } +func testUniqueNullsNotDistinctIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE public.products ( + id INTEGER PRIMARY KEY, + product_name VARCHAR(100), + serial_number TEXT, + UNIQUE NULLS NOT DISTINCT (product_name, serial_number) + );`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "syntax error", uniqueNullsNotDistinctIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -348,4 +365,7 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "foreign key referenced partitioned table", ybVersion), testForeignKeyReferencesPartitionedTableIssue) assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "unique nulls not distinct", ybVersion), testUniqueNullsNotDistinctIssue) + assert.True(t, success) } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index defde49879..e452f34a38 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -394,6 +394,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), NewJsonbSubscriptingDetector(query, p.jsonbColumns, p.getJsonbReturnTypeFunctions()), + NewUniqueNullsNotDistinctDetector(query), NewJsonPredicateExprDetector(query), } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e13597625b..97e364f922 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -163,6 +163,14 @@ CHECK (xpath_exists('/invoice/customer', data));` WITH (security_invoker = true) AS SELECT employee_id, first_name FROM public.employees;` + stmt21 = `CREATE TABLE public.products ( + id INTEGER PRIMARY KEY, + product_name VARCHAR(100), + serial_number TEXT, + UNIQUE NULLS NOT DISTINCT (product_name, serial_number) + );` + stmt22 = `ALTER TABLE public.products ADD CONSTRAINT unique_product_name UNIQUE NULLS NOT DISTINCT (product_name);` + stmt23 = `CREATE UNIQUE INDEX unique_email_idx ON users (email) NULLS NOT DISTINCT;` ) func modifiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { @@ -286,6 +294,15 @@ func TestDDLIssues(t *testing.T) { stmt20: []QueryIssue{ NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), }, + stmt21: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt21), + }, + stmt22: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt22), + }, + stmt23: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("INDEX", "unique_email_idx ON users", stmt23), + }, } for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -300,7 +317,7 @@ func TestDDLIssues(t *testing.T) { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { return cmp.Equal(expectedIssue, queryIssue) }) - assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + assert.True(t, found, "Expected issue not found: %v in statement: %s. \nFound: %v", expectedIssue, stmt, issues) } } } diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 73d76b7e1b..7889fb9529 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -420,6 +420,30 @@ func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) { return selectStmtNode, nil } +func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { + protoMsg, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + indexStmtNode, ok := protoMsg.(*pg_query.IndexStmt) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEXSTMT_NODE) + } + return indexStmtNode, nil +} + +func ProtoAsTableConstraint(msg protoreflect.Message) (*pg_query.Constraint, error) { + proto, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + constraintNode, ok := proto.(*pg_query.Constraint) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_CONSTRAINT_NODE) + } + return constraintNode, nil +} + /* Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 9177afc10c..9bf2b2716f 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -54,6 +54,7 @@ const ( PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" + PG_QUERY_INDEXSTMT_NODE = "pg_query.IndexStmt" ) // function type for processing nodes during traversal From b9e614dd9fc60cae61f173da3d871e2900f5215c Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 9 Jan 2025 14:40:46 +0530 Subject: [PATCH 096/105] Send error messages to callhome (#2160) Send error messages in each phase to callhome as part of phase payload. --- yb-voyager/cmd/analyzeSchema.go | 5 +- yb-voyager/cmd/assessMigrationBulkCommand.go | 6 +-- yb-voyager/cmd/assessMigrationCommand.go | 5 +- yb-voyager/cmd/common.go | 46 +++++++++++------- yb-voyager/cmd/endMigrationCommand.go | 5 +- yb-voyager/cmd/export.go | 2 +- yb-voyager/cmd/exportData.go | 15 +++--- yb-voyager/cmd/exportDataFromTarget.go | 3 +- yb-voyager/cmd/exportDataStatus.go | 2 +- yb-voyager/cmd/exportSchema.go | 5 +- yb-voyager/cmd/importData.go | 9 ++-- yb-voyager/cmd/importDataFileCommand.go | 5 +- yb-voyager/cmd/importDataToSource.go | 3 +- yb-voyager/cmd/importDataToSourceReplica.go | 3 +- yb-voyager/cmd/importSchema.go | 10 ++-- yb-voyager/src/callhome/diagnostics.go | 51 +++++++++++++------- yb-voyager/src/callhome/diagnostics_test.go | 42 +++++++++------- yb-voyager/src/srcdb/postgres.go | 2 +- yb-voyager/src/srcdb/yugabytedb.go | 2 +- yb-voyager/src/utils/logging.go | 3 ++ 20 files changed, 134 insertions(+), 90 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index a002df9efa..53daef6dd9 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1151,7 +1151,7 @@ func analyzeSchema() { generateAnalyzeSchemaReport(msr, JSON) } - packAndSendAnalyzeSchemaPayload(COMPLETE) + packAndSendAnalyzeSchemaPayload(COMPLETE, "") schemaAnalysisReport := createSchemaAnalysisIterationCompletedEvent(schemaAnalysisReport) controlPlane.SchemaAnalysisIterationCompleted(&schemaAnalysisReport) @@ -1234,7 +1234,7 @@ var reasonsToSendObjectNameToCallhome = []string{ UNSUPPORTED_EXTENSION_ISSUE, } -func packAndSendAnalyzeSchemaPayload(status string) { +func packAndSendAnalyzeSchemaPayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -1273,6 +1273,7 @@ func packAndSendAnalyzeSchemaPayload(status string) { dbObject.ObjectNames = "" return dbObject })), + Error: callhome.SanitizeErrorMsg(errorMsg), } payload.PhasePayload = callhome.MarshalledJsonString(analyzePayload) payload.Status = status diff --git a/yb-voyager/cmd/assessMigrationBulkCommand.go b/yb-voyager/cmd/assessMigrationBulkCommand.go index fb44192f8f..af6acffc52 100644 --- a/yb-voyager/cmd/assessMigrationBulkCommand.go +++ b/yb-voyager/cmd/assessMigrationBulkCommand.go @@ -57,14 +57,13 @@ var assessMigrationBulkCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { err := assessMigrationBulk() if err != nil { - packAndSendAssessMigrationBulkPayload(ERROR) utils.ErrExit("failed assess migration bulk: %s", err) } - packAndSendAssessMigrationBulkPayload(COMPLETE) + packAndSendAssessMigrationBulkPayload(COMPLETE, "") }, } -func packAndSendAssessMigrationBulkPayload(status string) { +func packAndSendAssessMigrationBulkPayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -77,6 +76,7 @@ func packAndSendAssessMigrationBulkPayload(status string) { } assessMigBulkPayload := callhome.AssessMigrationBulkPhasePayload{ FleetConfigCount: len(bulkAssessmentDBConfigs), + Error: callhome.SanitizeErrorMsg(errorMsg), } payload.PhasePayload = callhome.MarshalledJsonString(assessMigBulkPayload) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 2774b98b44..c4a887585d 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -215,10 +215,9 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { IndexSizingStats: callhome.MarshalledJsonString(indexSizingStats), SchemaSummary: callhome.MarshalledJsonString(schemaSummaryCopy), IopsInterval: intervalForCapturingIOPS, + Error: callhome.SanitizeErrorMsg(errMsg), } - if status == ERROR { - assessPayload.Error = "ERROR" // removing error for now, TODO to see if we want to keep it - } + if assessmentMetadataDirFlag == "" { sourceDBDetails := callhome.SourceDBDetails{ DBType: source.DBType, diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index cc55f737a1..e7ed70a94d 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1413,31 +1413,41 @@ func PackAndSendCallhomePayloadOnExit() { if callHomeErrorOrCompletePayloadSent { return } + + var errorMsg string + var status string + if utils.ErrExitErr != nil { + errorMsg = utils.ErrExitErr.Error() + status = ERROR + } else { + status = EXIT + } + switch currentCommand { case assessMigrationCmd.CommandPath(): - packAndSendAssessMigrationPayload(EXIT, "Exiting....") + packAndSendAssessMigrationPayload(status, errorMsg) case assessMigrationBulkCmd.CommandPath(): - packAndSendAssessMigrationBulkPayload(EXIT) + packAndSendAssessMigrationBulkPayload(status, errorMsg) case exportSchemaCmd.CommandPath(): - packAndSendExportSchemaPayload(EXIT) + packAndSendExportSchemaPayload(status, errorMsg) case analyzeSchemaCmd.CommandPath(): - packAndSendAnalyzeSchemaPayload(EXIT) + packAndSendAnalyzeSchemaPayload(status, errorMsg) case importSchemaCmd.CommandPath(): - packAndSendImportSchemaPayload(EXIT, "Exiting....") + packAndSendImportSchemaPayload(status, errorMsg) case exportDataCmd.CommandPath(), exportDataFromSrcCmd.CommandPath(): - packAndSendExportDataPayload(EXIT) + packAndSendExportDataPayload(status, errorMsg) case exportDataFromTargetCmd.CommandPath(): - packAndSendExportDataFromTargetPayload(EXIT) + packAndSendExportDataFromTargetPayload(status, errorMsg) case importDataCmd.CommandPath(), importDataToTargetCmd.CommandPath(): - packAndSendImportDataPayload(EXIT) + packAndSendImportDataPayload(status, errorMsg) case importDataToSourceCmd.CommandPath(): - packAndSendImportDataToSourcePayload(EXIT) + packAndSendImportDataToSourcePayload(status, errorMsg) case importDataToSourceReplicaCmd.CommandPath(): - packAndSendImportDataToSrcReplicaPayload(EXIT) + packAndSendImportDataToSrcReplicaPayload(status, errorMsg) case endMigrationCmd.CommandPath(): - packAndSendEndMigrationPayload(EXIT) + packAndSendEndMigrationPayload(status, errorMsg) case importDataFileCmd.CommandPath(): - packAndSendImportDataFilePayload(EXIT) + packAndSendImportDataFilePayload(status, errorMsg) } } @@ -1499,17 +1509,17 @@ func sendCallhomePayloadAtIntervals() { time.Sleep(15 * time.Minute) switch currentCommand { case exportDataCmd.CommandPath(), exportDataFromSrcCmd.CommandPath(): - packAndSendExportDataPayload(INPROGRESS) + packAndSendExportDataPayload(INPROGRESS, "") case exportDataFromTargetCmd.CommandPath(): - packAndSendExportDataFromTargetPayload(INPROGRESS) + packAndSendExportDataFromTargetPayload(INPROGRESS, "") case importDataCmd.CommandPath(), importDataToTargetCmd.CommandPath(): - packAndSendImportDataPayload(INPROGRESS) + packAndSendImportDataPayload(INPROGRESS, "") case importDataToSourceCmd.CommandPath(): - packAndSendImportDataToSourcePayload(INPROGRESS) + packAndSendImportDataToSourcePayload(INPROGRESS, "") case importDataToSourceReplicaCmd.CommandPath(): - packAndSendImportDataToSrcReplicaPayload(INPROGRESS) + packAndSendImportDataToSrcReplicaPayload(INPROGRESS, "") case importDataFileCmd.CommandPath(): - packAndSendImportDataFilePayload(INPROGRESS) + packAndSendImportDataFilePayload(INPROGRESS, "") } } } diff --git a/yb-voyager/cmd/endMigrationCommand.go b/yb-voyager/cmd/endMigrationCommand.go index f016c8cbd2..14704d5553 100644 --- a/yb-voyager/cmd/endMigrationCommand.go +++ b/yb-voyager/cmd/endMigrationCommand.go @@ -97,10 +97,10 @@ func endMigrationCommandFn(cmd *cobra.Command, args []string) { cleanupExportDir() utils.PrintAndLog("Migration ended successfully") - packAndSendEndMigrationPayload(COMPLETE) + packAndSendEndMigrationPayload(COMPLETE, "") } -func packAndSendEndMigrationPayload(status string) { +func packAndSendEndMigrationPayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -115,6 +115,7 @@ func packAndSendEndMigrationPayload(status string) { BackupLogFiles: bool(backupLogFiles), BackupSchemaFiles: bool(backupSchemaFiles), SaveMigrationReports: bool(saveMigrationReports), + Error: callhome.SanitizeErrorMsg(errorMsg), } payload.PhasePayload = callhome.MarshalledJsonString(endMigrationPayload) payload.Status = status diff --git a/yb-voyager/cmd/export.go b/yb-voyager/cmd/export.go index 516b80d206..746461e5e5 100644 --- a/yb-voyager/cmd/export.go +++ b/yb-voyager/cmd/export.go @@ -299,7 +299,7 @@ func validateSSLMode() { if source.DBType == ORACLE || slices.Contains(validSSLModes[source.DBType], source.SSLMode) { return } else { - utils.ErrExit("Error: Invalid sslmode: %q. Valid SSL modes are %v", validSSLModes[source.DBType]) + utils.ErrExit("Error: Invalid sslmode: %q. Valid SSL modes are %v", source.SSLMode, validSSLModes[source.DBType]) } } diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index 3016017843..973f129b3d 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -129,7 +129,7 @@ func exportDataCommandFn(cmd *cobra.Command, args []string) { success := exportData() if success { - sendPayloadAsPerExporterRole(COMPLETE) + sendPayloadAsPerExporterRole(COMPLETE, "") setDataIsExported() color.Green("Export of data complete") @@ -140,24 +140,24 @@ func exportDataCommandFn(cmd *cobra.Command, args []string) { } else { color.Red("Export of data failed! Check %s/logs for more details.", exportDir) log.Error("Export of data failed.") - sendPayloadAsPerExporterRole(ERROR) + sendPayloadAsPerExporterRole(ERROR, "") atexit.Exit(1) } } -func sendPayloadAsPerExporterRole(status string) { +func sendPayloadAsPerExporterRole(status string, errorMsg string) { if !callhome.SendDiagnostics { return } switch exporterRole { case SOURCE_DB_EXPORTER_ROLE: - packAndSendExportDataPayload(status) + packAndSendExportDataPayload(status, errorMsg) case TARGET_DB_EXPORTER_FB_ROLE, TARGET_DB_EXPORTER_FF_ROLE: - packAndSendExportDataFromTargetPayload(status) + packAndSendExportDataFromTargetPayload(status, errorMsg) } } -func packAndSendExportDataPayload(status string) { +func packAndSendExportDataPayload(status string, errorMsg string) { if !shouldSendCallhome() { return @@ -182,6 +182,7 @@ func packAndSendExportDataPayload(status string) { exportDataPayload := callhome.ExportDataPhasePayload{ ParallelJobs: int64(source.NumConnections), StartClean: bool(startClean), + Error: callhome.SanitizeErrorMsg(errorMsg), } updateExportSnapshotDataStatsInPayload(&exportDataPayload) @@ -1046,7 +1047,7 @@ func extractTableListFromString(fullTableList []sqlname.NameTuple, flagTableList result := lo.Filter(fullTableList, func(tableName sqlname.NameTuple, _ int) bool { ok, err := tableName.MatchesPattern(pattern) if err != nil { - utils.ErrExit("Invalid table name pattern %q: %s", err) + utils.ErrExit("Invalid table name pattern %q: %s", pattern, err) } return ok }) diff --git a/yb-voyager/cmd/exportDataFromTarget.go b/yb-voyager/cmd/exportDataFromTarget.go index 38c557933a..31fe1a248d 100644 --- a/yb-voyager/cmd/exportDataFromTarget.go +++ b/yb-voyager/cmd/exportDataFromTarget.go @@ -112,7 +112,7 @@ func initSourceConfFromTargetConf() error { return nil } -func packAndSendExportDataFromTargetPayload(status string) { +func packAndSendExportDataFromTargetPayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -128,6 +128,7 @@ func packAndSendExportDataFromTargetPayload(status string) { exportDataPayload := callhome.ExportDataPhasePayload{ ParallelJobs: int64(source.NumConnections), StartClean: bool(startClean), + Error: callhome.SanitizeErrorMsg(errorMsg), } exportDataPayload.Phase = exportPhase diff --git a/yb-voyager/cmd/exportDataStatus.go b/yb-voyager/cmd/exportDataStatus.go index e33468ae7b..5e8b87c742 100644 --- a/yb-voyager/cmd/exportDataStatus.go +++ b/yb-voyager/cmd/exportDataStatus.go @@ -262,7 +262,7 @@ func startExportPB(progressContainer *mpb.Progress, mapKey string, quitChan chan time.Sleep(100 * time.Millisecond) break } else if err != nil { //error other than EOF - utils.ErrExit("Error while reading file %s: %v", tableDataFile, err) + utils.ErrExit("Error while reading file %s: %v", tableDataFile.Name(), err) } if isDataLine(line, source.DBType, &insideCopyStmt) { tableMetadata.CountLiveRows += 1 diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index f84888a8fd..4c8f82d720 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -175,7 +175,7 @@ func exportSchema() error { utils.PrintAndLog("\nExported schema files created under directory: %s\n\n", filepath.Join(exportDir, "schema")) - packAndSendExportSchemaPayload(COMPLETE) + packAndSendExportSchemaPayload(COMPLETE, "") saveSourceDBConfInMSR() setSchemaIsExported() @@ -185,7 +185,7 @@ func exportSchema() error { return nil } -func packAndSendExportSchemaPayload(status string) { +func packAndSendExportSchemaPayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -203,6 +203,7 @@ func packAndSendExportSchemaPayload(status string) { AppliedRecommendations: assessmentRecommendationsApplied, UseOrafce: bool(source.UseOrafce), CommentsOnObjects: bool(source.CommentsOnObjects), + Error: callhome.SanitizeErrorMsg(errorMsg), } payload.PhasePayload = callhome.MarshalledJsonString(exportSchemaPayload) diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 99e71c969b..33da3047d0 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -693,11 +693,11 @@ func importData(importFileTasks []*ImportFileTask) { case TARGET_DB_IMPORTER_ROLE: importDataCompletedEvent := createSnapshotImportCompletedEvent() controlPlane.SnapshotImportCompleted(&importDataCompletedEvent) - packAndSendImportDataPayload(COMPLETE) + packAndSendImportDataPayload(COMPLETE, "") case SOURCE_REPLICA_DB_IMPORTER_ROLE: - packAndSendImportDataToSrcReplicaPayload(COMPLETE) + packAndSendImportDataToSrcReplicaPayload(COMPLETE, "") case SOURCE_DB_IMPORTER_ROLE: - packAndSendImportDataToSourcePayload(COMPLETE) + packAndSendImportDataToSourcePayload(COMPLETE, "") } } @@ -755,7 +755,7 @@ func waitForDebeziumStartIfRequired() error { return nil } -func packAndSendImportDataPayload(status string) { +func packAndSendImportDataPayload(status string, errorMsg string) { if !shouldSendCallhome() { return @@ -774,6 +774,7 @@ func packAndSendImportDataPayload(status string) { ParallelJobs: int64(tconf.Parallelism), StartClean: bool(startClean), EnableUpsert: bool(tconf.EnableUpsert), + Error: callhome.SanitizeErrorMsg(errorMsg), } //Getting the imported snapshot details diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 0deaee34b0..9af7e99f18 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -96,7 +96,7 @@ var importDataFileCmd = &cobra.Command{ utils.ErrExit("failed to get migration UUID: %w", err) } importData(importFileTasks) - packAndSendImportDataFilePayload(COMPLETE) + packAndSendImportDataFilePayload(COMPLETE, "") }, PostRun: func(cmd *cobra.Command, args []string) { @@ -330,7 +330,7 @@ func checkAndParseEscapeAndQuoteChar() { } -func packAndSendImportDataFilePayload(status string) { +func packAndSendImportDataFilePayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -350,6 +350,7 @@ func packAndSendImportDataFilePayload(status string) { ParallelJobs: int64(tconf.Parallelism), StartClean: bool(startClean), DataFileParameters: callhome.MarshalledJsonString(dataFileParameters), + Error: callhome.SanitizeErrorMsg(errorMsg), } switch true { case strings.Contains(dataDir, "s3://"): diff --git a/yb-voyager/cmd/importDataToSource.go b/yb-voyager/cmd/importDataToSource.go index a14e5b458d..fc27566953 100644 --- a/yb-voyager/cmd/importDataToSource.go +++ b/yb-voyager/cmd/importDataToSource.go @@ -87,7 +87,7 @@ func initTargetConfFromSourceConf() error { return nil } -func packAndSendImportDataToSourcePayload(status string) { +func packAndSendImportDataToSourcePayload(status string, errorMsg string) { if !shouldSendCallhome() { return @@ -107,6 +107,7 @@ func packAndSendImportDataToSourcePayload(status string) { ParallelJobs: int64(tconf.Parallelism), StartClean: bool(startClean), LiveWorkflowType: FALL_BACK, + Error: callhome.SanitizeErrorMsg(errorMsg), } importDataPayload.Phase = importPhase diff --git a/yb-voyager/cmd/importDataToSourceReplica.go b/yb-voyager/cmd/importDataToSourceReplica.go index 4c68550167..f8e93ac236 100644 --- a/yb-voyager/cmd/importDataToSourceReplica.go +++ b/yb-voyager/cmd/importDataToSourceReplica.go @@ -92,7 +92,7 @@ func updateFallForwardEnabledInMetaDB() { } } -func packAndSendImportDataToSrcReplicaPayload(status string) { +func packAndSendImportDataToSrcReplicaPayload(status string, errorMsg string) { if !shouldSendCallhome() { return } @@ -111,6 +111,7 @@ func packAndSendImportDataToSrcReplicaPayload(status string) { ParallelJobs: int64(tconf.Parallelism), StartClean: bool(startClean), LiveWorkflowType: FALL_FORWARD, + Error: callhome.SanitizeErrorMsg(errorMsg), } importRowsMap, err := getImportedSnapshotRowsMap("source-replica") if err != nil { diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index 388535a2e3..250309618e 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -279,12 +279,9 @@ func packAndSendImportSchemaPayload(status string, errMsg string) { parts := strings.Split(stmt, "*/\n") errorsList = append(errorsList, strings.Trim(parts[0], "/*\n")) //trimming the prefix of `/*\n` from parts[0] (the error msg) } - if status == ERROR { - errorsList = append(errorsList, errMsg) - } else { - if len(errorsList) > 0 && status != EXIT { - payload.Status = COMPLETE_WITH_ERRORS - } + + if len(errorsList) > 0 && status != EXIT { + payload.Status = COMPLETE_WITH_ERRORS } //import-schema specific payload details @@ -296,6 +293,7 @@ func packAndSendImportSchemaPayload(status string, errMsg string) { ErrorCount: len(errorsList), PostSnapshotImport: bool(flagPostSnapshotImport), StartClean: bool(startClean), + Error: callhome.SanitizeErrorMsg(errMsg), } payload.PhasePayload = callhome.MarshalledJsonString(importSchemaPayload) err := callhome.SendPayload(&payload) diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 8e53134162..9f4d40e7bd 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -24,6 +24,7 @@ import ( "os" "reflect" "strconv" + "strings" "github.com/google/uuid" "github.com/samber/lo" @@ -111,7 +112,7 @@ type AssessMigrationPhasePayload struct { UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` MigrationCaveats string `json:"migration_caveats"` UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` - Error string `json:"error,omitempty"` // Removed it for now, TODO + Error string `json:"error"` TableSizingStats string `json:"table_sizing_stats"` IndexSizingStats string `json:"index_sizing_stats"` SchemaSummary string `json:"schema_summary"` @@ -120,7 +121,8 @@ type AssessMigrationPhasePayload struct { } type AssessMigrationBulkPhasePayload struct { - FleetConfigCount int `json:"fleet_config_count"` // Not storing any source info just the count of db configs passed to bulk cmd + FleetConfigCount int `json:"fleet_config_count"` // Not storing any source info just the count of db configs passed to bulk cmd + Error string `json:"error"` } type ObjectSizingStats struct { @@ -132,10 +134,11 @@ type ObjectSizingStats struct { } type ExportSchemaPhasePayload struct { - StartClean bool `json:"start_clean"` - AppliedRecommendations bool `json:"applied_recommendations"` - UseOrafce bool `json:"use_orafce"` - CommentsOnObjects bool `json:"comments_on_objects"` + StartClean bool `json:"start_clean"` + AppliedRecommendations bool `json:"applied_recommendations"` + UseOrafce bool `json:"use_orafce"` + CommentsOnObjects bool `json:"comments_on_objects"` + Error string `json:"error"` } // SHOULD NOT REMOVE THESE TWO (issues, database_objects) FIELDS of AnalyzePhasePayload as parsing these specifically here @@ -144,6 +147,7 @@ type AnalyzePhasePayload struct { TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` Issues string `json:"issues"` DatabaseObjects string `json:"database_objects"` + Error string `json:"error"` } type ExportDataPhasePayload struct { ParallelJobs int64 `json:"parallel_jobs"` @@ -156,16 +160,18 @@ type ExportDataPhasePayload struct { TotalExportedEvents int64 `json:"total_exported_events,omitempty"` EventsExportRate int64 `json:"events_export_rate_3m,omitempty"` LiveWorkflowType string `json:"live_workflow_type,omitempty"` + Error string `json:"error"` } type ImportSchemaPhasePayload struct { - ContinueOnError bool `json:"continue_on_error"` - EnableOrafce bool `json:"enable_orafce"` - IgnoreExist bool `json:"ignore_exist"` - RefreshMviews bool `json:"refresh_mviews"` - ErrorCount int `json:"errors"` // changing it to count of errors only - PostSnapshotImport bool `json:"post_snapshot_import"` - StartClean bool `json:"start_clean"` + ContinueOnError bool `json:"continue_on_error"` + EnableOrafce bool `json:"enable_orafce"` + IgnoreExist bool `json:"ignore_exist"` + RefreshMviews bool `json:"refresh_mviews"` + ErrorCount int `json:"errors"` // changing it to count of errors only + PostSnapshotImport bool `json:"post_snapshot_import"` + StartClean bool `json:"start_clean"` + Error string `json:"error"` } type ImportDataPhasePayload struct { @@ -179,6 +185,7 @@ type ImportDataPhasePayload struct { EventsImportRate int64 `json:"events_import_rate_3m,omitempty"` LiveWorkflowType string `json:"live_workflow_type,omitempty"` EnableUpsert bool `json:"enable_upsert"` + Error string `json:"error"` } type ImportDataFilePhasePayload struct { @@ -188,6 +195,7 @@ type ImportDataFilePhasePayload struct { FileStorageType string `json:"file_storage_type"` StartClean bool `json:"start_clean"` DataFileParameters string `json:"data_file_parameters"` + Error string `json:"error"` } type DataFileParameters struct { @@ -200,10 +208,11 @@ type DataFileParameters struct { } type EndMigrationPhasePayload struct { - BackupDataFiles bool `json:"backup_data_files"` - BackupLogFiles bool `json:"backup_log_files"` - BackupSchemaFiles bool `json:"backup_schema_files"` - SaveMigrationReports bool `json:"save_migration_reports"` + BackupDataFiles bool `json:"backup_data_files"` + BackupLogFiles bool `json:"backup_log_files"` + BackupSchemaFiles bool `json:"backup_schema_files"` + SaveMigrationReports bool `json:"save_migration_reports"` + Error string `json:"error"` } var DoNotStoreFlags = []string{ @@ -283,3 +292,11 @@ func SendPayload(payload *Payload) error { return nil } + +// We want to ensure that no user-specific information is sent to the call-home service. +// Therefore, we only send the segment of the error message before the first ":" as that is the generic error message. +// Note: This is a temporary solution. A better solution would be to have +// properly structured errors and only send the generic error message to callhome. +func SanitizeErrorMsg(errorMsg string) string { + return strings.Split(errorMsg, ":")[0] +} diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index 84781c01fa..b20b6a27de 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -92,7 +92,7 @@ func TestCallhomeStructs(t *testing.T) { UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` MigrationCaveats string `json:"migration_caveats"` UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` - Error string `json:"error,omitempty"` + Error string `json:"error"` TableSizingStats string `json:"table_sizing_stats"` IndexSizingStats string `json:"index_sizing_stats"` SchemaSummary string `json:"schema_summary"` @@ -104,7 +104,8 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate AssessMigrationBulkPhasePayload Struct Definition", actualType: reflect.TypeOf(AssessMigrationBulkPhasePayload{}), expectedType: struct { - FleetConfigCount int `json:"fleet_config_count"` + FleetConfigCount int `json:"fleet_config_count"` + Error string `json:"error"` }{}, }, { @@ -122,10 +123,11 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate ExportSchemaPhasePayload Struct Definition", actualType: reflect.TypeOf(ExportSchemaPhasePayload{}), expectedType: struct { - StartClean bool `json:"start_clean"` - AppliedRecommendations bool `json:"applied_recommendations"` - UseOrafce bool `json:"use_orafce"` - CommentsOnObjects bool `json:"comments_on_objects"` + StartClean bool `json:"start_clean"` + AppliedRecommendations bool `json:"applied_recommendations"` + UseOrafce bool `json:"use_orafce"` + CommentsOnObjects bool `json:"comments_on_objects"` + Error string `json:"error"` }{}, }, { @@ -135,6 +137,7 @@ func TestCallhomeStructs(t *testing.T) { TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` Issues string `json:"issues"` DatabaseObjects string `json:"database_objects"` + Error string `json:"error"` }{}, }, { @@ -150,19 +153,21 @@ func TestCallhomeStructs(t *testing.T) { TotalExportedEvents int64 `json:"total_exported_events,omitempty"` EventsExportRate int64 `json:"events_export_rate_3m,omitempty"` LiveWorkflowType string `json:"live_workflow_type,omitempty"` + Error string `json:"error"` }{}, }, { name: "Validate ImportSchemaPhasePayload Struct Definition", actualType: reflect.TypeOf(ImportSchemaPhasePayload{}), expectedType: struct { - ContinueOnError bool `json:"continue_on_error"` - EnableOrafce bool `json:"enable_orafce"` - IgnoreExist bool `json:"ignore_exist"` - RefreshMviews bool `json:"refresh_mviews"` - ErrorCount int `json:"errors"` - PostSnapshotImport bool `json:"post_snapshot_import"` - StartClean bool `json:"start_clean"` + ContinueOnError bool `json:"continue_on_error"` + EnableOrafce bool `json:"enable_orafce"` + IgnoreExist bool `json:"ignore_exist"` + RefreshMviews bool `json:"refresh_mviews"` + ErrorCount int `json:"errors"` + PostSnapshotImport bool `json:"post_snapshot_import"` + StartClean bool `json:"start_clean"` + Error string `json:"error"` }{}, }, { @@ -178,6 +183,7 @@ func TestCallhomeStructs(t *testing.T) { EventsImportRate int64 `json:"events_import_rate_3m,omitempty"` LiveWorkflowType string `json:"live_workflow_type,omitempty"` EnableUpsert bool `json:"enable_upsert"` + Error string `json:"error"` }{}, }, { @@ -190,6 +196,7 @@ func TestCallhomeStructs(t *testing.T) { FileStorageType string `json:"file_storage_type"` StartClean bool `json:"start_clean"` DataFileParameters string `json:"data_file_parameters"` + Error string `json:"error"` }{}, }, { @@ -208,10 +215,11 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate EndMigrationPhasePayload Struct Definition", actualType: reflect.TypeOf(EndMigrationPhasePayload{}), expectedType: struct { - BackupDataFiles bool `json:"backup_data_files"` - BackupLogFiles bool `json:"backup_log_files"` - BackupSchemaFiles bool `json:"backup_schema_files"` - SaveMigrationReports bool `json:"save_migration_reports"` + BackupDataFiles bool `json:"backup_data_files"` + BackupLogFiles bool `json:"backup_log_files"` + BackupSchemaFiles bool `json:"backup_schema_files"` + SaveMigrationReports bool `json:"save_migration_reports"` + Error string `json:"error"` }{}, }, } diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 5d7a83813a..58c4d142a2 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -785,7 +785,7 @@ WHERE parent.relname='%s' AND nmsp_parent.nspname = '%s' `, tname, sname) rows, err := pg.db.Query(query) if err != nil { log.Errorf("failed to list partitions of table %s: query = [ %s ], error = %s", tableName, query, err) - utils.ErrExit("failed to find the partitions for table %s:", tableName, err) + utils.ErrExit("failed to find the partitions for table %s: %v", tableName, err) } defer func() { closeErr := rows.Close() diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index c2c7ef07e5..2a90f8f052 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -756,7 +756,7 @@ WHERE parent.relname='%s' AND nmsp_parent.nspname = '%s' `, tname, sname) rows, err := yb.db.Query(query) if err != nil { log.Errorf("failed to list partitions of table %s: query = [ %s ], error = %s", tableName, query, err) - utils.ErrExit("failed to find the partitions for table %s:", tableName, err) + utils.ErrExit("failed to find the partitions for table %s: %v", tableName, err) } defer func() { closeErr := rows.Close() diff --git a/yb-voyager/src/utils/logging.go b/yb-voyager/src/utils/logging.go index 74e2562bd4..c4dd17b0b0 100644 --- a/yb-voyager/src/utils/logging.go +++ b/yb-voyager/src/utils/logging.go @@ -24,7 +24,10 @@ import ( "github.com/tebeka/atexit" ) +var ErrExitErr error + func ErrExit(formatString string, args ...interface{}) { + ErrExitErr = fmt.Errorf(formatString, args...) formatString = strings.Replace(formatString, "%w", "%s", -1) fmt.Fprintf(os.Stderr, formatString+"\n", args...) log.Errorf(formatString+"\n", args...) From 95cc4989ee21d6486934e6e63d702b68b1329ff8 Mon Sep 17 00:00:00 2001 From: Shubham Dabriwala Date: Thu, 9 Jan 2025 15:25:27 +0530 Subject: [PATCH 097/105] Make export-dir, source and target database names dynamic to enable parallel runs (#2019) --- .github/workflows/mysql-migtests.yml | 1 + migtests/lib/yb.py | 3 +- migtests/scripts/add-pk-from-alter-to-create | 2 +- migtests/scripts/functions.sh | 85 ++++++++++++++++--- .../scripts/live-migration-fallb-run-test.sh | 20 +++-- .../scripts/live-migration-fallf-run-test.sh | 20 +++-- migtests/scripts/live-migration-run-test.sh | 13 ++- migtests/scripts/run-schema-migration.sh | 2 +- migtests/scripts/run-test-export-data.sh | 2 +- migtests/scripts/run-test.sh | 22 ++++- .../scripts/run-validate-assessment-report.sh | 2 +- .../run-validate-bulk-assessment-report.sh | 3 + migtests/scripts/yugabytedb/env.sh | 9 +- migtests/tests/mysql/basic-live-test/env.sh | 3 +- migtests/tests/mysql/chinook/Chinook.sql | 1 - migtests/tests/mysql/chinook/env.sh | 6 +- migtests/tests/mysql/datatypes/env.sh | 1 - migtests/tests/mysql/sakila/env.sh | 1 + migtests/tests/mysql/sakila/init-db | 2 +- migtests/tests/oracle/basic-live-test/env.sh | 2 - .../oracle/bulk-assessment-test/cleanup-db | 11 +++ .../tests/oracle/bulk-assessment-test/init-db | 15 ++-- .../case-sensitivity-reserved-words/env.sh | 2 - migtests/tests/oracle/co-db/env.sh | 1 + migtests/tests/oracle/partitions/env.sh | 2 - migtests/tests/oracle/sequences/env.sh | 2 - .../oracle/sequences/sequence_schema.sql | 2 +- .../oracle/sequences/validateAfterChanges | 4 +- .../oracle/unique-key-conflicts-test/env.sh | 2 - migtests/tests/pg/datatypes/env.sh | 1 - .../tests/pg/partitions-with-indexes/env.sh | 2 +- migtests/tests/pg/partitions/env.sh | 1 - migtests/tests/pg/partitions/fix-schema | 18 ++-- .../tests/pg/unique-key-conflicts-test/env.sh | 2 - ...yager-oracle-gather-assessment-metadata.sh | 2 +- 35 files changed, 184 insertions(+), 83 deletions(-) create mode 100644 migtests/tests/oracle/bulk-assessment-test/cleanup-db diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 81c7ff2ae8..1dfda76a62 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -137,6 +137,7 @@ jobs: # Placeholder for now so that a basic test can run - name: Create the live migration user + if: always() run: | mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';' diff --git a/migtests/lib/yb.py b/migtests/lib/yb.py index 854de33fe4..c9351928d8 100644 --- a/migtests/lib/yb.py +++ b/migtests/lib/yb.py @@ -52,7 +52,8 @@ def new_source_db(): def verify_colocation(tgt, source_db_type): print("Verifying the colocation of the tables") - json_file = "export-dir/assessment/reports/migration_assessment_report.json" + export_dir = os.getenv("EXPORT_DIR", "export-dir") + json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json" sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file) diff --git a/migtests/scripts/add-pk-from-alter-to-create b/migtests/scripts/add-pk-from-alter-to-create index 83d0ef3eb2..77f1ab5b7b 100755 --- a/migtests/scripts/add-pk-from-alter-to-create +++ b/migtests/scripts/add-pk-from-alter-to-create @@ -5,7 +5,7 @@ import os import re import shutil -table_file_path = os.getenv('TEST_DIR')+'/export-dir/schema/tables/table.sql' +table_file_path = os.getenv('EXPORT_DIR')+'/schema/tables/table.sql' #copy the table_file_path to table_file_path+'.old' src = table_file_path diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 4d8e1d6f4f..3b80a1ea7c 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -134,7 +134,8 @@ grant_user_permission_oracle(){ */ GRANT FLASHBACK ANY TABLE TO ybvoyager; EOF - run_sqlplus_as_sys ${db_name} "oracle-inputs.sql" + run_sqlplus_as_sys ${db_name} "oracle-inputs.sql" + rm oracle-inputs.sql } @@ -149,6 +150,8 @@ EOF run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql" cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql" + rm create-pdb-tablespace.sql + rm oracle-inputs.sql } grant_permissions_for_live_migration_pg() { @@ -191,7 +194,7 @@ run_sqlplus_as_sys() { run_sqlplus_as_schema_owner() { db_name=$1 sql=$2 - conn_string="${SOURCE_DB_USER_SCHEMA_OWNER}/${SOURCE_DB_USER_SCHEMA_OWNER_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" + conn_string="${SOURCE_DB_SCHEMA}/${SOURCE_DB_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" echo exit | sqlplus -f "${conn_string}" @"${sql}" } @@ -623,16 +626,6 @@ get_value_from_msr(){ echo $val } -create_ff_schema(){ - db_name=$1 - - cat > create-ff-schema.sql << EOF - CREATE USER FF_SCHEMA IDENTIFIED BY "password"; - GRANT all privileges to FF_SCHEMA; -EOF - run_sqlplus_as_sys ${db_name} "create-ff-schema.sql" -} - set_replica_identity(){ db_schema=$1 cat > alter_replica_identity.sql < $TEMP_SCRIPT + + run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT + + # Clean up the temporary file after execution + rm -f $TEMP_SCRIPT elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${SOURCE_DB_NAME}" psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql @@ -1057,3 +1059,60 @@ cutover_to_target() { yb-voyager initiate cutover to target ${args} $* } + +create_source_db() { + source_db=$1 + case ${SOURCE_DB_TYPE} in + postgresql) + run_psql postgres "DROP DATABASE IF EXISTS ${source_db};" + run_psql postgres "CREATE DATABASE ${source_db};" + ;; + mysql) + run_mysql mysql "DROP DATABASE IF EXISTS ${source_db};" + run_mysql mysql "CREATE DATABASE ${source_db};" + ;; + oracle) + cat > create-oracle-schema.sql << EOF + CREATE USER ${source_db} IDENTIFIED BY "password"; + GRANT all privileges to ${source_db}; +EOF + run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-oracle-schema.sql" + rm create-oracle-schema.sql + ;; + *) + echo "ERROR: Source DB not created for ${SOURCE_DB_TYPE}" + exit 1 + ;; + esac +} + +normalize_and_export_vars() { + local test_suffix=$1 + + # Normalize TEST_NAME + # Keeping the full name for PG and MySQL to test out large schema/export dir names + export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" + + # Set EXPORT_DIR + export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"} + if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" + fi + + # Set database-specific variables + case "${SOURCE_DB_TYPE}" in + postgresql|mysql) + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"} + ;; + oracle) + # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit + # Since test_suffix is the unique identifying factor, we need to add it post all the normalization + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} + ;; + *) + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + ;; + esac +} diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index fc4fb886a5..bac1a1e968 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -18,9 +18,7 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 - export PYTHONPATH="${REPO_ROOT}/migtests/lib" # Order of env.sh import matters. @@ -37,10 +35,12 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -source ${SCRIPTS}/yugabytedb/env.sh - source ${SCRIPTS}/functions.sh +normalize_and_export_vars "fallb" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" @@ -56,6 +56,10 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [ "${SOURCE_DB_TYPE}" = "oracle" ] + then + create_source_db ${SOURCE_DB_SCHEMA} + fi ./init-db step "Grant source database user permissions for live migration" @@ -169,7 +173,11 @@ main() { import_schema --post-snapshot-import true --refresh-mviews=true step "Run snapshot validations." - "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' + "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' || { + tail_log_file "yb-voyager-import-data.log" + tail_log_file "yb-voyager-export-data-from-source.log" + exit 1 + } step "Inserting new events" run_sql_file source_delta.sql @@ -264,7 +272,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index 175630f70a..54dbe7bd78 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" export PATH="${PATH}:/usr/lib/oracle/21/client64/bin" @@ -38,12 +37,14 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi +source ${SCRIPTS}/functions.sh + +normalize_and_export_vars "fallf" + source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh source ${SCRIPTS}/yugabytedb/env.sh -source ${SCRIPTS}/functions.sh - main() { echo "Deleting the parent export-dir present in the test directory" @@ -62,7 +63,9 @@ main() { if [ "${SOURCE_DB_TYPE}" = "oracle" ] then - create_ff_schema ${SOURCE_REPLICA_DB_NAME} + create_source_db ${SOURCE_DB_SCHEMA} + # TODO: Add dynamic Fall Forward schema creation. Currently using the same name for all tests. + create_source_db ${SOURCE_REPLICA_DB_SCHEMA} run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql fi ./init-db @@ -206,7 +209,12 @@ main() { import_schema --post-snapshot-import true --refresh-mviews true step "Run snapshot validations." - "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' + "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' || { + tail_log_file "yb-voyager-import-data.log" + tail_log_file "yb-voyager-export-data-from-source.log" + tail_log_file "yb-voyager-import-data-to-source-replica.log" + exit 1 + } step "Inserting new events to source" run_sql_file source_delta.sql @@ -286,7 +294,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 1594f1449c..98f99e6beb 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -37,10 +36,12 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -source ${SCRIPTS}/yugabytedb/env.sh - source ${SCRIPTS}/functions.sh +normalize_and_export_vars "live" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" @@ -56,6 +57,10 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [ "${SOURCE_DB_TYPE}" = "oracle" ] + then + create_source_db ${SOURCE_DB_SCHEMA} + fi ./init-db step "Grant source database user permissions for live migration" @@ -218,7 +223,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-schema-migration.sh b/migtests/scripts/run-schema-migration.sh index 0f37bab501..8a142cf152 100755 --- a/migtests/scripts/run-schema-migration.sh +++ b/migtests/scripts/run-schema-migration.sh @@ -139,7 +139,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-test-export-data.sh b/migtests/scripts/run-test-export-data.sh index 04e84bdd22..dcbeecd362 100755 --- a/migtests/scripts/run-test-export-data.sh +++ b/migtests/scripts/run-test-export-data.sh @@ -74,7 +74,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index f0d9408cf8..e40cc60d10 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -17,7 +17,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -33,11 +32,16 @@ then else source ${TEST_DIR}/env.sh fi + source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh -source ${SCRIPTS}/yugabytedb/env.sh + source ${SCRIPTS}/functions.sh +normalize_and_export_vars "offline" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" rm -rf ${EXPORT_DIR} @@ -52,6 +56,18 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [[ "${SKIP_DB_CREATION}" != "true" ]]; then + if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + create_source_db "${SOURCE_DB_NAME}" + elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + create_source_db "${SOURCE_DB_SCHEMA}" + else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + fi + else + echo "Skipping database creation as SKIP_DB_CREATION is set to true." + fi ./init-db step "Grant source database user permissions" @@ -190,7 +206,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-validate-assessment-report.sh b/migtests/scripts/run-validate-assessment-report.sh index 5cdac1d392..75e3d8875c 100755 --- a/migtests/scripts/run-validate-assessment-report.sh +++ b/migtests/scripts/run-validate-assessment-report.sh @@ -94,7 +94,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" } main diff --git a/migtests/scripts/run-validate-bulk-assessment-report.sh b/migtests/scripts/run-validate-bulk-assessment-report.sh index 05b43c87ab..ac67773b1d 100755 --- a/migtests/scripts/run-validate-bulk-assessment-report.sh +++ b/migtests/scripts/run-validate-bulk-assessment-report.sh @@ -54,6 +54,7 @@ main() { echo "Assigning permissions to the export-dir to execute init-db, cleanup-db scripts" chmod +x ${TEST_DIR}/init-db + chmod +x ${TEST_DIR}/cleanup-db step "START: ${TEST_NAME}" print_env @@ -109,6 +110,8 @@ main() { fi step "Clean up" + ./cleanup-db + rm -rf "${BULK_ASSESSMENT_DIR}" } diff --git a/migtests/scripts/yugabytedb/env.sh b/migtests/scripts/yugabytedb/env.sh index a288b14ffc..10b9e70e5f 100644 --- a/migtests/scripts/yugabytedb/env.sh +++ b/migtests/scripts/yugabytedb/env.sh @@ -7,7 +7,10 @@ export TARGET_DB_ADMIN_PASSWORD=${TARGET_DB_ADMIN_PASSWORD:-''} export TARGET_DB_SCHEMA=${TARGET_DB_SCHEMA:-'public'} # The PG driver, used to connect to YB, is case-sensitive about database name. -if [ "${TARGET_DB_NAME}" == "" ] -then - export TARGET_DB_NAME=`echo ${SOURCE_DB_NAME} | tr [A-Z] [a-z]` +if [ "${TARGET_DB_NAME}" == "" ]; then + if [ "${SOURCE_DB_TYPE}" == "oracle" ]; then + export TARGET_DB_NAME=$(echo ${SOURCE_DB_SCHEMA} | tr [A-Z] [a-z]) + else + export TARGET_DB_NAME=$(echo ${SOURCE_DB_NAME} | tr [A-Z] [a-z]) + fi fi diff --git a/migtests/tests/mysql/basic-live-test/env.sh b/migtests/tests/mysql/basic-live-test/env.sh index dfe82e6660..62ddf80b40 100644 --- a/migtests/tests/mysql/basic-live-test/env.sh +++ b/migtests/tests/mysql/basic-live-test/env.sh @@ -1,2 +1 @@ -export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"live_test"} \ No newline at end of file +export SOURCE_DB_TYPE="mysql" \ No newline at end of file diff --git a/migtests/tests/mysql/chinook/Chinook.sql b/migtests/tests/mysql/chinook/Chinook.sql index c761f0d041..f3309ff09b 100644 --- a/migtests/tests/mysql/chinook/Chinook.sql +++ b/migtests/tests/mysql/chinook/Chinook.sql @@ -15829,4 +15829,3 @@ INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (17, 2096); INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (17, 3290); INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (18, 597); - diff --git a/migtests/tests/mysql/chinook/env.sh b/migtests/tests/mysql/chinook/env.sh index ff571a1c0e..c640e171cd 100644 --- a/migtests/tests/mysql/chinook/env.sh +++ b/migtests/tests/mysql/chinook/env.sh @@ -1,3 +1,5 @@ export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"Chinook"} -export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" \ No newline at end of file +export SOURCE_DB_NAME="Chinook" +export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" +export SKIP_DB_CREATION="true" + diff --git a/migtests/tests/mysql/datatypes/env.sh b/migtests/tests/mysql/datatypes/env.sh index 1d4759f090..aedc7cffd6 100644 --- a/migtests/tests/mysql/datatypes/env.sh +++ b/migtests/tests/mysql/datatypes/env.sh @@ -1,2 +1 @@ export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"datatypes"} diff --git a/migtests/tests/mysql/sakila/env.sh b/migtests/tests/mysql/sakila/env.sh index 620640ea43..5fa4dc4acb 100644 --- a/migtests/tests/mysql/sakila/env.sh +++ b/migtests/tests/mysql/sakila/env.sh @@ -1,2 +1,3 @@ export SOURCE_DB_TYPE="mysql" export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"sakila"} +export SKIP_DB_CREATION="true" \ No newline at end of file diff --git a/migtests/tests/mysql/sakila/init-db b/migtests/tests/mysql/sakila/init-db index f6eb9c5bd6..b783d909f1 100755 --- a/migtests/tests/mysql/sakila/init-db +++ b/migtests/tests/mysql/sakila/init-db @@ -23,4 +23,4 @@ run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-data.sql;" run_mysql ${SOURCE_DB_NAME} "ALTER TABLE address DROP COLUMN location;" echo "Check source database." -run_mysql ${SOURCE_DB_NAME} "SELECT count(*) FROM payment;" +run_mysql ${SOURCE_DB_NAME} "SELECT count(*) FROM payment;" \ No newline at end of file diff --git a/migtests/tests/oracle/basic-live-test/env.sh b/migtests/tests/oracle/basic-live-test/env.sh index 722796b6a8..998f6081f6 100644 --- a/migtests/tests/oracle/basic-live-test/env.sh +++ b/migtests/tests/oracle/basic-live-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema" diff --git a/migtests/tests/oracle/bulk-assessment-test/cleanup-db b/migtests/tests/oracle/bulk-assessment-test/cleanup-db new file mode 100644 index 0000000000..07ebc07a8e --- /dev/null +++ b/migtests/tests/oracle/bulk-assessment-test/cleanup-db @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +set -x + +source ${SCRIPTS}/functions.sh + +for schema in TEST_SCHEMA TEST_SCHEMA2; do + export SOURCE_DB_SCHEMA=${schema} + run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema +done \ No newline at end of file diff --git a/migtests/tests/oracle/bulk-assessment-test/init-db b/migtests/tests/oracle/bulk-assessment-test/init-db index b8acab5175..0cd3340a0d 100755 --- a/migtests/tests/oracle/bulk-assessment-test/init-db +++ b/migtests/tests/oracle/bulk-assessment-test/init-db @@ -5,21 +5,20 @@ set -x source ${SCRIPTS}/functions.sh -echo "Deleting existing data in source database" -run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema +export SOURCE_DB_SCHEMA="TEST_SCHEMA" + +create_source_db ${SOURCE_DB_SCHEMA} + +./cleanup-db echo "Initialising source database 1 & inserting data" run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../assessment-report-test/oracle_assessment_report.sql echo "Initialising source database 2 & inserting data" -cat > create-schema2.sql << EOF - CREATE USER TEST_SCHEMA2 identified by "password"; - GRANT ALL PRIVILEGES TO TEST_SCHEMA2; -EOF - run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-schema2.sql" +export SOURCE_DB_SCHEMA="TEST_SCHEMA2" -export SOURCE_DB_USER_SCHEMA_OWNER="TEST_SCHEMA2" +create_source_db ${SOURCE_DB_SCHEMA} run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../partitions/partition_schema.sql run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../partitions/partition_data.sql diff --git a/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh b/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh index 4f724929c1..429f19fb3a 100644 --- a/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh +++ b/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema2" diff --git a/migtests/tests/oracle/co-db/env.sh b/migtests/tests/oracle/co-db/env.sh index f0f431a669..15d6f15d90 100644 --- a/migtests/tests/oracle/co-db/env.sh +++ b/migtests/tests/oracle/co-db/env.sh @@ -1,4 +1,5 @@ export SOURCE_DB_TYPE="oracle" export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"CO"} export TARGET_DB_NAME=${TARGET_DB_NAME:-"co_db_test"} +export SKIP_DB_CREATION="true" diff --git a/migtests/tests/oracle/partitions/env.sh b/migtests/tests/oracle/partitions/env.sh index 05489d8a99..515cf7abbe 100644 --- a/migtests/tests/oracle/partitions/env.sh +++ b/migtests/tests/oracle/partitions/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"partition_test"} diff --git a/migtests/tests/oracle/sequences/env.sh b/migtests/tests/oracle/sequences/env.sh index ef3b4aa439..0357fe1d0e 100644 --- a/migtests/tests/oracle/sequences/env.sh +++ b/migtests/tests/oracle/sequences/env.sh @@ -1,3 +1 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"oracle_sequences_test"} diff --git a/migtests/tests/oracle/sequences/sequence_schema.sql b/migtests/tests/oracle/sequences/sequence_schema.sql index 1baac1c7fc..310edc2501 100644 --- a/migtests/tests/oracle/sequences/sequence_schema.sql +++ b/migtests/tests/oracle/sequences/sequence_schema.sql @@ -61,7 +61,7 @@ CREATE TABLE empty_identity_def ( description VARCHAR2(100) not null ); - drop table "Case_Sensitive_always"; +drop table "Case_Sensitive_always"; CREATE TABLE "Case_Sensitive_always" ( id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, diff --git a/migtests/tests/oracle/sequences/validateAfterChanges b/migtests/tests/oracle/sequences/validateAfterChanges index cc6c2705e3..95f0832da3 100755 --- a/migtests/tests/oracle/sequences/validateAfterChanges +++ b/migtests/tests/oracle/sequences/validateAfterChanges @@ -166,14 +166,14 @@ def migration_completed_checks_yb(): def migration_completed_checks_ff(): print("Running tests on Oracle source replica") global db_schema - db_schema = os.environ.get("SOURCE_REPLICA_DB_SCHEMA") + db_schema = os.environ.get("SOURCE_REPLICA_DB_SCHEMA").upper() change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source_replica") def migration_completed_checks_fb(): print("Running tests on Oracle source") global db_schema - db_schema = os.environ.get("SOURCE_DB_SCHEMA") + db_schema = os.environ.get("SOURCE_DB_SCHEMA").upper() change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source") diff --git a/migtests/tests/oracle/unique-key-conflicts-test/env.sh b/migtests/tests/oracle/unique-key-conflicts-test/env.sh index 722796b6a8..998f6081f6 100644 --- a/migtests/tests/oracle/unique-key-conflicts-test/env.sh +++ b/migtests/tests/oracle/unique-key-conflicts-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema" diff --git a/migtests/tests/pg/datatypes/env.sh b/migtests/tests/pg/datatypes/env.sh index 328c0615f3..3606e020cc 100644 --- a/migtests/tests/pg/datatypes/env.sh +++ b/migtests/tests/pg/datatypes/env.sh @@ -1,3 +1,2 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"pg_datatypes"} export SOURCE_DB_SCHEMA="public" diff --git a/migtests/tests/pg/partitions-with-indexes/env.sh b/migtests/tests/pg/partitions-with-indexes/env.sh index 6b22305323..7f216dc8da 100644 --- a/migtests/tests/pg/partitions-with-indexes/env.sh +++ b/migtests/tests/pg/partitions-with-indexes/env.sh @@ -1,3 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} +export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions_with_index"} export SOURCE_DB_SCHEMA="public,p1,p2" diff --git a/migtests/tests/pg/partitions/env.sh b/migtests/tests/pg/partitions/env.sh index 9ba7757697..bbca0e5017 100644 --- a/migtests/tests/pg/partitions/env.sh +++ b/migtests/tests/pg/partitions/env.sh @@ -1,4 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} export SOURCE_DB_SCHEMA="public,p1,p2" export MOVE_PK_FROM_ALTER_TO_CREATE="true" \ No newline at end of file diff --git a/migtests/tests/pg/partitions/fix-schema b/migtests/tests/pg/partitions/fix-schema index 57007db3f5..8a341cc602 100755 --- a/migtests/tests/pg/partitions/fix-schema +++ b/migtests/tests/pg/partitions/fix-schema @@ -3,15 +3,15 @@ set -e set -x -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/schema/tables/table.sql -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/schema/tables/table.sql -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/schema/tables/table.sql +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/schema/tables/table.sql +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/schema/tables/table.sql +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/schema/tables/table.sql -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql # Added so that the validations work(to check the recommendations in target YB applied or not) -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json \ No newline at end of file +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json \ No newline at end of file diff --git a/migtests/tests/pg/unique-key-conflicts-test/env.sh b/migtests/tests/pg/unique-key-conflicts-test/env.sh index f812fc984c..4615850a1d 100644 --- a/migtests/tests/pg/unique-key-conflicts-test/env.sh +++ b/migtests/tests/pg/unique-key-conflicts-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"unique_key_conflict_cases"} export SOURCE_DB_SCHEMA="public,non_public" -export SOURCE_REPLICA_DB_NAME="unique_key_conflict_cases_replica" \ No newline at end of file diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index 7550d8f765..5f30e93410 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -269,7 +269,7 @@ main() { run_command "$ora2pg_cmd" done - ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet --quiet > $assessment_metadata_dir/schema/ora2pg_report.csv" + ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet --quiet > $assessment_metadata_dir/schema/ora2pg_report.csv" log "INFO" "executing ora2pg command for report: $ora2pg_report_cmd" run_command "$ora2pg_report_cmd" From 073fdac7fce05af1d6ccf5be53962bd3074052a5 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 9 Jan 2025 15:40:16 +0530 Subject: [PATCH 098/105] Modify error message strings. (#2164) Modify error strings to include user-specific details only after a ":". This is because as part of sending error messages to callhome, we sanitize them by picking the segment of the error message before the first ":" --- yb-voyager/cmd/analyzeSchema.go | 12 ++--- yb-voyager/cmd/archiveCommand.go | 4 +- yb-voyager/cmd/assessMigrationBulkCommand.go | 6 +-- yb-voyager/cmd/assessMigrationCommand.go | 4 +- yb-voyager/cmd/common.go | 20 ++++----- yb-voyager/cmd/endMigrationCommand.go | 8 ++-- yb-voyager/cmd/export.go | 2 +- yb-voyager/cmd/exportData.go | 22 +++++----- yb-voyager/cmd/exportDataStatus.go | 2 +- yb-voyager/cmd/exportDataStatusCommand.go | 8 ++-- .../cmd/getDataMigrationReportCommand.go | 8 ++-- yb-voyager/cmd/importData.go | 44 +++++++++---------- yb-voyager/cmd/importDataFileCommand.go | 6 +-- yb-voyager/cmd/importDataStatusCommand.go | 2 +- yb-voyager/cmd/importSchema.go | 14 +++--- yb-voyager/cmd/importSchemaYugabyteDB.go | 6 +-- yb-voyager/cmd/root.go | 4 +- yb-voyager/src/datastore/azDatastore.go | 4 +- yb-voyager/src/datastore/gcsDatastore.go | 12 ++--- yb-voyager/src/datastore/localDatastore.go | 2 +- yb-voyager/src/datastore/s3Datastore.go | 6 +-- yb-voyager/src/dbzm/status.go | 2 +- yb-voyager/src/srcdb/common.go | 2 +- yb-voyager/src/srcdb/mysql.go | 10 ++--- yb-voyager/src/srcdb/ora2pg.go | 2 +- yb-voyager/src/srcdb/ora2pg_export_data.go | 10 ++--- yb-voyager/src/srcdb/oracle.go | 22 +++++----- yb-voyager/src/srcdb/pg_dump_export_data.go | 4 +- .../src/srcdb/pg_dump_extract_schema.go | 4 +- yb-voyager/src/srcdb/postgres.go | 28 ++++++------ yb-voyager/src/srcdb/srcdb.go | 2 +- yb-voyager/src/srcdb/yugabytedb.go | 38 ++++++++-------- yb-voyager/src/tgtdb/oracle.go | 8 ++-- yb-voyager/src/tgtdb/postgres.go | 6 +-- yb-voyager/src/tgtdb/yugabytedb.go | 10 ++--- yb-voyager/src/utils/az/azutils.go | 2 +- yb-voyager/src/utils/s3/s3utils.go | 2 +- 37 files changed, 174 insertions(+), 174 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 53daef6dd9..8ff11381a5 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -329,14 +329,14 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { } err = parserIssueDetector.ParseRequiredDDLs(sqlStmtInfo.formattedStmt) if err != nil { - utils.ErrExit("error parsing stmt[%s]: %v", sqlStmtInfo.formattedStmt, err) + utils.ErrExit("error parsing stmt: [%s]: %v", sqlStmtInfo.formattedStmt, err) } if parserIssueDetector.IsGinIndexPresentInSchema { summaryMap["INDEX"].details[GIN_INDEX_DETAILS] = true } ddlIssues, err := parserIssueDetector.GetDDLIssues(sqlStmtInfo.formattedStmt, targetDbVersion) if err != nil { - utils.ErrExit("error getting ddl issues for stmt[%s]: %v", sqlStmtInfo.formattedStmt, err) + utils.ErrExit("error getting ddl issues for stmt: [%s]: %v", sqlStmtInfo.formattedStmt, err) } for _, i := range ddlIssues { schemaAnalysisReport.Issues = append(schemaAnalysisReport.Issues, convertIssueInstanceToAnalyzeIssue(i, fpath, false)) @@ -852,7 +852,7 @@ func parseSqlFileForObjectType(path string, objType string) []sqlInfo { reportNextSql := 0 file, err := os.ReadFile(path) if err != nil { - utils.ErrExit("Error while reading %q: %s", path, err) + utils.ErrExit("Error while reading file: %q: %s", path, err) } lines := strings.Split(string(file), "\n") @@ -1098,7 +1098,7 @@ func checkConversions(sqlInfoArr []sqlInfo, filePath string) { for _, sqlStmtInfo := range sqlInfoArr { parseTree, err := pg_query.Parse(sqlStmtInfo.stmt) if err != nil { - utils.ErrExit("failed to parse the stmt %v: %v", sqlStmtInfo.stmt, err) + utils.ErrExit("failed to parse the stmt: %v: %v", sqlStmtInfo.stmt, err) } createConvNode, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_CreateConversionStmt) @@ -1202,7 +1202,7 @@ func generateAnalyzeSchemaReport(msr *metadb.MigrationStatusRecord, reportFormat file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - utils.ErrExit("Error while opening %q: %s", reportPath, err) + utils.ErrExit("Error while opening: %q: %s", reportPath, err) } defer func() { if err := file.Close(); err != nil { @@ -1212,7 +1212,7 @@ func generateAnalyzeSchemaReport(msr *metadb.MigrationStatusRecord, reportFormat _, err = file.WriteString(finalReport) if err != nil { - utils.ErrExit("failed to write report to %q: %s", reportPath, err) + utils.ErrExit("failed to write report to: %q: %s", reportPath, err) } fmt.Printf("-- find schema analysis report at: %s\n", reportPath) return nil diff --git a/yb-voyager/cmd/archiveCommand.go b/yb-voyager/cmd/archiveCommand.go index 8fac243a68..5dfec23227 100644 --- a/yb-voyager/cmd/archiveCommand.go +++ b/yb-voyager/cmd/archiveCommand.go @@ -59,12 +59,12 @@ func validateCommonArchiveFlags() { func validateMoveToFlag() { if moveDestination != "" { if !utils.FileOrFolderExists(moveDestination) { - utils.ErrExit("move destination %q doesn't exists.\n", moveDestination) + utils.ErrExit("move destination doesn't exists: %q: \n", moveDestination) } else { var err error moveDestination, err = filepath.Abs(moveDestination) if err != nil { - utils.ErrExit("Failed to get absolute path for move destination %q: %v\n", moveDestination, err) + utils.ErrExit("Failed to get absolute path for move destination: %q: %v\n", moveDestination, err) } moveDestination = filepath.Clean(moveDestination) fmt.Printf("Note: Using %q as move destination\n", moveDestination) diff --git a/yb-voyager/cmd/assessMigrationBulkCommand.go b/yb-voyager/cmd/assessMigrationBulkCommand.go index af6acffc52..d758c2e922 100644 --- a/yb-voyager/cmd/assessMigrationBulkCommand.go +++ b/yb-voyager/cmd/assessMigrationBulkCommand.go @@ -50,7 +50,7 @@ var assessMigrationBulkCmd = &cobra.Command{ PreRun: func(cmd *cobra.Command, args []string) { err := validateFleetConfigFile(fleetConfigPath) if err != nil { - utils.ErrExit("%s", err.Error()) + utils.ErrExit("validating fleet config file: %s", err.Error()) } }, @@ -458,7 +458,7 @@ func validateBulkAssessmentDirFlag() { utils.ErrExit(`ERROR: required flag "bulk-assessment-dir" not set`) } if !utils.FileOrFolderExists(bulkAssessmentDir) { - utils.ErrExit("bulk-assessment-dir %q doesn't exists.\n", bulkAssessmentDir) + utils.ErrExit("bulk-assessment-dir doesn't exists: %q\n", bulkAssessmentDir) } else { if bulkAssessmentDir == "." { fmt.Println("Note: Using current directory as bulk-assessment-dir") @@ -466,7 +466,7 @@ func validateBulkAssessmentDirFlag() { var err error bulkAssessmentDir, err = filepath.Abs(bulkAssessmentDir) if err != nil { - utils.ErrExit("Failed to get absolute path for bulk-assessment-dir %q: %v\n", exportDir, err) + utils.ErrExit("Failed to get absolute path for bulk-assessment-dir: %q: %v\n", exportDir, err) } bulkAssessmentDir = filepath.Clean(bulkAssessmentDir) } diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index c4a887585d..357c900ff4 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -648,7 +648,7 @@ func checkStartCleanForAssessMigration(metadataDirPassedByUser bool) { utils.ErrExit("failed to start clean: %v", err) } } else { - utils.ErrExit("assessment metadata or reports files already exist in the assessment directory at '%s'. Use the --start-clean flag to clear the directory before proceeding.", assessmentDir) + utils.ErrExit("assessment metadata or reports files already exist in the assessment directory: '%s'. Use the --start-clean flag to clear the directory before proceeding.", assessmentDir) } } } @@ -1713,7 +1713,7 @@ func validateSourceDBTypeForAssessMigration() { func validateAssessmentMetadataDirFlag() { if assessmentMetadataDirFlag != "" { if !utils.FileOrFolderExists(assessmentMetadataDirFlag) { - utils.ErrExit("assessment metadata directory %q provided with `--assessment-metadata-dir` flag does not exist", assessmentMetadataDirFlag) + utils.ErrExit("assessment metadata directory: %q provided with `--assessment-metadata-dir` flag does not exist", assessmentMetadataDirFlag) } else { log.Infof("using provided assessment metadata directory: %s", assessmentMetadataDirFlag) } diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index e7ed70a94d..7c9394d432 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -165,7 +165,7 @@ func getMappingForTableNameVsTableFileName(dataDirPath string, noWait bool) map[ fullTableName := fmt.Sprintf("%s.%s", schemaName, tableName) table, err := namereg.NameReg.LookupTableName(fullTableName) if err != nil { - utils.ErrExit("lookup table %s in name registry : %v", fullTableName, err) + utils.ErrExit("lookup table in name registry: %q: %v", fullTableName, err) } tableNameVsFileNameMap[table.ForKey()] = fileName } @@ -173,7 +173,7 @@ func getMappingForTableNameVsTableFileName(dataDirPath string, noWait bool) map[ tocTextFileDataBytes, err := os.ReadFile(tocTextFilePath) if err != nil { - utils.ErrExit("Failed to read file %q: %v", tocTextFilePath, err) + utils.ErrExit("Failed to read file: %q: %v", tocTextFilePath, err) } tocTextFileData := strings.Split(string(tocTextFileDataBytes), "\n") @@ -208,7 +208,7 @@ func GetTableRowCount(filePath string) map[string]int64 { fileBytes, err := os.ReadFile(filePath) if err != nil { - utils.ErrExit("read file %q: %s", filePath, err) + utils.ErrExit("read file: %q: %s", filePath, err) } lines := strings.Split(strings.Trim(string(fileBytes), "\n"), "\n") @@ -294,7 +294,7 @@ func displayExportedRowCountSnapshot(snapshotViaDebezium bool) { for _, key := range keys { table, err := namereg.NameReg.LookupTableName(key) if err != nil { - utils.ErrExit("lookup table %s in name registry : %v", key, err) + utils.ErrExit("lookup table in name registry: %q: %v", key, err) } displayTableName := table.CurrentName.Unqualified.MinQuoted //Using the ForOutput() as a key for leafPartitions map as we are populating the map in that way. @@ -328,7 +328,7 @@ func displayExportedRowCountSnapshot(snapshotViaDebezium bool) { } table, err := namereg.NameReg.LookupTableName(fmt.Sprintf("%s.%s", tableStatus.SchemaName, tableStatus.TableName)) if err != nil { - utils.ErrExit("lookup table %s in name registry : %v", tableStatus.TableName, err) + utils.ErrExit("lookup table in name registry : %q: %v", tableStatus.TableName, err) } displayTableName := table.CurrentName.Unqualified.MinQuoted partitions := leafPartitions[table.ForOutput()] @@ -388,7 +388,7 @@ func displayImportedRowCountSnapshot(state *ImportDataState, tasks []*ImportFile for _, tableName := range tableList { tableRowCount, err := state.GetImportedSnapshotRowCountForTable(tableName) if err != nil { - utils.ErrExit("could not fetch snapshot row count for table %q: %w", tableName, err) + utils.ErrExit("could not fetch snapshot row count for table: %q: %w", tableName, err) } snapshotRowCount.Put(tableName, tableRowCount) } @@ -441,7 +441,7 @@ func CreateMigrationProjectIfNotExists(dbType string, exportDir string) { for _, subdir := range projectSubdirs { err := exec.Command("mkdir", "-p", filepath.Join(projectDirPath, subdir)).Run() if err != nil { - utils.ErrExit("couldn't create sub-directories under %q: %v", projectDirPath, err) + utils.ErrExit("couldn't create sub-directories under: %q: %v", projectDirPath, err) } } @@ -458,7 +458,7 @@ func CreateMigrationProjectIfNotExists(dbType string, exportDir string) { err := exec.Command("mkdir", "-p", filepath.Join(schemaDir, databaseObjectDirName)).Run() if err != nil { - utils.ErrExit("couldn't create sub-directories under %q: %v", schemaDir, err) + utils.ErrExit("couldn't create sub-directories under: %q: %v", schemaDir, err) } } @@ -929,7 +929,7 @@ func renameTableIfRequired(table string) (string, bool) { } defaultSchema, noDefaultSchema := GetDefaultPGSchema(schema, "|") if noDefaultSchema && len(strings.Split(table, ".")) <= 1 { - utils.ErrExit("no default schema found to qualify table %s", table) + utils.ErrExit("no default schema found to qualify table: %s", table) } tableName := sqlname.NewSourceNameFromMaybeQualifiedName(table, defaultSchema) fromTable := tableName.Qualified.Unquoted @@ -937,7 +937,7 @@ func renameTableIfRequired(table string) (string, bool) { if renameTablesMap[fromTable] != "" { tableTup, err := namereg.NameReg.LookupTableName(renameTablesMap[fromTable]) if err != nil { - utils.ErrExit("lookup failed for the table %s", renameTablesMap[fromTable]) + utils.ErrExit("lookup failed for the table: %s", renameTablesMap[fromTable]) } return tableTup.ForMinOutput(), true diff --git a/yb-voyager/cmd/endMigrationCommand.go b/yb-voyager/cmd/endMigrationCommand.go index 14704d5553..c161d79a87 100644 --- a/yb-voyager/cmd/endMigrationCommand.go +++ b/yb-voyager/cmd/endMigrationCommand.go @@ -664,7 +664,7 @@ func cleanupExportDir() { for _, subdir := range subdirs { err := os.RemoveAll(filepath.Join(exportDir, subdir)) if err != nil { - utils.ErrExit("removing %s directory: %v", subdir, err) + utils.ErrExit("removing directory: %q: %v", subdir, err) } } } @@ -785,7 +785,7 @@ func stopVoyagerCommand(lockFile *lockfile.Lockfile, signal syscall.Signal) { ongoingCmd := lockFile.GetCmdName() ongoingCmdPID, err := lockFile.GetCmdPID() if err != nil { - utils.ErrExit("getting PID of ongoing voyager command %q: %v", ongoingCmd, err) + utils.ErrExit("getting PID of ongoing voyager command: %q: %v", ongoingCmd, err) } fmt.Printf("stopping the ongoing command: %s\n", ongoingCmd) @@ -811,7 +811,7 @@ func stopDataExportCommand(lockFile *lockfile.Lockfile) { ongoingCmd := lockFile.GetCmdName() ongoingCmdPID, err := lockFile.GetCmdPID() if err != nil { - utils.ErrExit("getting PID of ongoing voyager command %q: %v", ongoingCmd, err) + utils.ErrExit("getting PID of ongoing voyager command: %q: %v", ongoingCmd, err) } fmt.Printf("stopping the ongoing command: %s\n", ongoingCmd) @@ -830,7 +830,7 @@ func areOnDifferentFileSystems(path1 string, path2 string) bool { err2 := syscall.Stat(path2, &stat2) if err1 != nil || err2 != nil { - utils.ErrExit("getting file system info for %s and %s: %v, %v", path1, path2, err1, err2) + utils.ErrExit("getting file system info: for %s and %s: %v, %v", path1, path2, err1, err2) } return stat1.Dev != stat2.Dev diff --git a/yb-voyager/cmd/export.go b/yb-voyager/cmd/export.go index 746461e5e5..7e7488f4cb 100644 --- a/yb-voyager/cmd/export.go +++ b/yb-voyager/cmd/export.go @@ -299,7 +299,7 @@ func validateSSLMode() { if source.DBType == ORACLE || slices.Contains(validSSLModes[source.DBType], source.SSLMode) { return } else { - utils.ErrExit("Error: Invalid sslmode: %q. Valid SSL modes are %v", source.SSLMode, validSSLModes[source.DBType]) + utils.ErrExit("Invalid sslmode: %q. Valid SSL modes are %v", source.SSLMode, validSSLModes[source.DBType]) } } diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index 973f129b3d..0620700223 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -237,7 +237,7 @@ func exportData() bool { res := source.DB().CheckSchemaExists() if !res { - utils.ErrExit("schema %q does not exist", source.Schema) + utils.ErrExit("schema does not exist : %q", source.Schema) } if source.RunGuardrailsChecks { @@ -301,7 +301,7 @@ func exportData() bool { //Fine to lookup directly as this will root table in case of partitions tuple, err := namereg.NameReg.LookupTableName(renamedTable) if err != nil { - utils.ErrExit("lookup table name %s: %v", renamedTable, err) + utils.ErrExit("lookup table name: %s: %v", renamedTable, err) } currPartitions, ok := leafPartitions.Get(tuple) if !ok { @@ -759,7 +759,7 @@ func getInitialTableList() (map[string]string, []sqlname.NameTuple) { if parent == "" { tuple, err = namereg.NameReg.LookupTableName(fmt.Sprintf("%s.%s", schema, table)) if err != nil { - utils.ErrExit("lookup for table name %s failed err: %v", table, err) + utils.ErrExit("lookup for table name failed err: %s: %v", table, err) } } fullTableList = append(fullTableList, tuple) @@ -868,8 +868,8 @@ func exportDataOffline(ctx context.Context, cancel context.CancelFunc, finalTabl log.Infoln("Cancel() being called, within exportDataOffline()") cancel() //will cancel/stop both dump tool and progress bar time.Sleep(time.Second * 5) //give sometime for the cancel to complete before this function returns - utils.ErrExit("yb-voyager encountered internal error. "+ - "Check %s/logs/yb-voyager-export-data.log for more details.", exportDir) + utils.ErrExit("yb-voyager encountered internal error: "+ + "Check: %s/logs/yb-voyager-export-data.log for more details.", exportDir) } }() @@ -896,7 +896,7 @@ func exportDataOffline(ctx context.Context, cancel context.CancelFunc, finalTabl for _, seq := range sequenceList { seqTuple, err := namereg.NameReg.LookupTableName(seq) if err != nil { - utils.ErrExit("lookup for sequence %s failed err: %v", seq, err) + utils.ErrExit("lookup for sequence failed: %s: err: %v", seq, err) } finalTableList = append(finalTableList, seqTuple) } @@ -1019,7 +1019,7 @@ func clearMigrationStateIfRequired() { dbzm.IsMigrationInStreamingMode(exportDir) { utils.PrintAndLog("Continuing streaming from where we left off...") } else { - utils.ErrExit("%s/data directory is not empty, use --start-clean flag to clean the directories and start", exportDir) + utils.ErrExit("data directory is not empty, use --start-clean flag to clean the directories and start: %s", exportDir) } } } @@ -1047,7 +1047,7 @@ func extractTableListFromString(fullTableList []sqlname.NameTuple, flagTableList result := lo.Filter(fullTableList, func(tableName sqlname.NameTuple, _ int) bool { ok, err := tableName.MatchesPattern(pattern) if err != nil { - utils.ErrExit("Invalid table name pattern %q: %s", pattern, err) + utils.ErrExit("Invalid table name pattern: %q: %s", pattern, err) } return ok }) @@ -1064,7 +1064,7 @@ func extractTableListFromString(fullTableList []sqlname.NameTuple, flagTableList } if len(unknownTableNames) > 0 { utils.PrintAndLog("Unknown table names %v in the %s list", unknownTableNames, listName) - utils.ErrExit("Valid table names are %v", lo.Map(fullTableList, func(tableName sqlname.NameTuple, _ int) string { + utils.ErrExit("Valid table names are: %v", lo.Map(fullTableList, func(tableName sqlname.NameTuple, _ int) string { return tableName.ForOutput() })) } @@ -1143,13 +1143,13 @@ func startFallBackSetupIfRequired() { utils.PrintAndLog("Starting import data to source with command:\n %s", color.GreenString(cmdStr)) binary, lookErr := exec.LookPath(os.Args[0]) if lookErr != nil { - utils.ErrExit("could not find yb-voyager - %w", err) + utils.ErrExit("could not find yb-voyager: %w", err) } env := os.Environ() env = slices.Insert(env, 0, "SOURCE_DB_PASSWORD="+source.Password) execErr := syscall.Exec(binary, cmd, env) if execErr != nil { - utils.ErrExit("failed to run yb-voyager import data to source - %w\n Please re-run with command :\n%s", err, cmdStr) + utils.ErrExit("failed to run yb-voyager import data to source: %w\n Please re-run with command :\n%s", err, cmdStr) } } diff --git a/yb-voyager/cmd/exportDataStatus.go b/yb-voyager/cmd/exportDataStatus.go index 5e8b87c742..c68b102f3e 100644 --- a/yb-voyager/cmd/exportDataStatus.go +++ b/yb-voyager/cmd/exportDataStatus.go @@ -262,7 +262,7 @@ func startExportPB(progressContainer *mpb.Progress, mapKey string, quitChan chan time.Sleep(100 * time.Millisecond) break } else if err != nil { //error other than EOF - utils.ErrExit("Error while reading file %s: %v", tableDataFile.Name(), err) + utils.ErrExit("Error while reading file: %s: %v", tableDataFile.Name(), err) } if isDataLine(line, source.DBType, &insideCopyStmt) { tableMetadata.CountLiveRows += 1 diff --git a/yb-voyager/cmd/exportDataStatusCommand.go b/yb-voyager/cmd/exportDataStatusCommand.go index 78c9697b5e..3c3c310ff2 100644 --- a/yb-voyager/cmd/exportDataStatusCommand.go +++ b/yb-voyager/cmd/exportDataStatusCommand.go @@ -91,7 +91,7 @@ var exportDataStatusCmd = &cobra.Command{ reportFile := jsonfile.NewJsonFile[[]*exportTableMigStatusOutputRow](reportFilePath) err := reportFile.Create(&rows) if err != nil { - utils.ErrExit("creating into json file %s: %v", reportFilePath, err) + utils.ErrExit("creating into json file: %s: %v", reportFilePath, err) } fmt.Print(color.GreenString("Export data status report is written to %s\n", reportFilePath)) return @@ -123,7 +123,7 @@ func runExportDataStatusCmdDbzm(streamChanges bool, leafPartitions map[string][] exportStatusFilePath := filepath.Join(exportDir, "data", "export_status.json") status, err := dbzm.ReadExportStatus(exportStatusFilePath) if err != nil { - utils.ErrExit("Failed to read export status file %s: %v", exportStatusFilePath, err) + utils.ErrExit("Failed to read export status file: %s: %v", exportStatusFilePath, err) } if status == nil { return nil, fmt.Errorf("export data has not started yet. Try running after export has started") @@ -142,7 +142,7 @@ func runExportDataStatusCmdDbzm(streamChanges bool, leafPartitions map[string][] func getSnapshotExportStatusRow(tableStatus *dbzm.TableExportStatus, leafPartitions map[string][]string, msr *metadb.MigrationStatusRecord) *exportTableMigStatusOutputRow { nt, err := namereg.NameReg.LookupTableName(fmt.Sprintf("%s.%s", tableStatus.SchemaName, tableStatus.TableName)) if err != nil { - utils.ErrExit("lookup %s in name registry: %v", tableStatus.TableName, err) + utils.ErrExit("lookup in name registry: %s: %v", tableStatus.TableName, err) } //Using the ForOutput() as a key for leafPartitions map as we are populating the map in that way. displayTableName := getDisplayName(nt, leafPartitions[nt.ForOutput()], msr.IsExportTableListSet) @@ -183,7 +183,7 @@ func runExportDataStatusCmd(msr *metadb.MigrationStatusRecord, leafPartitions ma if errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("export data has not started yet. Try running after export has started") } - utils.ErrExit("Failed to read export status file %s: %v", exportSnapshotStatusFilePath, err) + utils.ErrExit("Failed to read export status file: %s: %v", exportSnapshotStatusFilePath, err) } exportedSnapshotRow, exportedSnapshotStatus, err := getExportedSnapshotRowsMap(exportStatusSnapshot) diff --git a/yb-voyager/cmd/getDataMigrationReportCommand.go b/yb-voyager/cmd/getDataMigrationReportCommand.go index 9a1a734cea..f37a5797cd 100644 --- a/yb-voyager/cmd/getDataMigrationReportCommand.go +++ b/yb-voyager/cmd/getDataMigrationReportCommand.go @@ -137,7 +137,7 @@ func getDataMigrationReportCmdFn(msr *metadb.MigrationStatusRecord) { exportStatusFilePath := filepath.Join(exportDir, "data", "export_status.json") dbzmStatus, err := dbzm.ReadExportStatus(exportStatusFilePath) if err != nil { - utils.ErrExit("Failed to read export status file %s: %v", exportStatusFilePath, err) + utils.ErrExit("Failed to read export status file: %s: %v", exportStatusFilePath, err) } if dbzmStatus == nil { utils.ErrExit("Export data has not started yet. Try running after export has started.") @@ -158,7 +158,7 @@ func getDataMigrationReportCmdFn(msr *metadb.MigrationStatusRecord) { if errors.Is(err, fs.ErrNotExist) { utils.ErrExit("Export data has not started yet. Try running after export has started.") } - utils.ErrExit("Failed to read export status file %s: %v", exportSnapshotStatusFilePath, err) + utils.ErrExit("Failed to read export status file: %s: %v", exportSnapshotStatusFilePath, err) } exportedPGSnapshotRowsMap, _, err = getExportedSnapshotRowsMap(exportSnapshotStatus) if err != nil { @@ -169,7 +169,7 @@ func getDataMigrationReportCmdFn(msr *metadb.MigrationStatusRecord) { tableName := fmt.Sprintf("%s.%s", tableExportStatus.SchemaName, tableExportStatus.TableName) nt, err := namereg.NameReg.LookupTableName(tableName) if err != nil { - utils.ErrExit("lookup %s in name registry: %v", tableName, err) + utils.ErrExit("lookup in name registry: %s: %v", tableName, err) } dbzmNameTupToRowCount.Put(nt, tableExportStatus.ExportedRowCountSnapshot) } @@ -306,7 +306,7 @@ func getDataMigrationReportCmdFn(msr *metadb.MigrationStatusRecord) { reportFile := jsonfile.NewJsonFile[[]*rowData](reportFilePath) err := reportFile.Create(&reportData) if err != nil { - utils.ErrExit("creating into json file %s: %v", reportFilePath, err) + utils.ErrExit("creating into json file: %s: %v", reportFilePath, err) } fmt.Print(color.GreenString("Data migration report is written to %s\n", reportFilePath)) return diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 33da3047d0..45050cf615 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -306,7 +306,7 @@ func startExportDataFromTargetIfRequired() { execErr := syscall.Exec(binary, cmd, env) if execErr != nil { - utils.ErrExit("failed to run yb-voyager export data from target - %w\n Please re-run with command :\n%s", execErr, cmdStr) + utils.ErrExit("failed to run yb-voyager export data from target: %w\n Please re-run with command :\n%s", execErr, cmdStr) } } @@ -419,7 +419,7 @@ func applyTableListFilter(importFileTasks []*ImportFileTask) []*ImportFileTask { } } if len(unqualifiedTables) > 0 { - utils.ErrExit("Qualify following table names %v in the %s list with schema-name.", unqualifiedTables, listName) + utils.ErrExit("Qualify following table names in the %s list with schema-name: %v", listName, unqualifiedTables) } log.Infof("%s tableList: %v", listName, result) return result, unknownTables @@ -846,7 +846,7 @@ func getIdentityColumnsForTables(tables []sqlname.NameTuple, identityType string for _, table := range tables { identityColumns, err := tdb.GetIdentityColumnNamesForTable(table, identityType) if err != nil { - utils.ErrExit("error in getting identity(%s) columns for table %s: %w", identityType, table, err) + utils.ErrExit("error in getting identity(%s) columns for table: %s: %w", identityType, table, err) } if len(identityColumns) > 0 { log.Infof("identity(%s) columns for table %s: %v", identityType, table, identityColumns) @@ -868,13 +868,13 @@ func getImportedProgressAmount(task *ImportFileTask, state *ImportDataState) int if reportProgressInBytes { byteCount, err := state.GetImportedByteCount(task.FilePath, task.TableNameTup) if err != nil { - utils.ErrExit("Failed to get imported byte count for table %s: %s", task.TableNameTup, err) + utils.ErrExit("Failed to get imported byte count for table: %s: %s", task.TableNameTup, err) } return byteCount } else { rowCount, err := state.GetImportedRowCount(task.FilePath, task.TableNameTup) if err != nil { - utils.ErrExit("Failed to get imported row count for table %s: %s", task.TableNameTup, err) + utils.ErrExit("Failed to get imported row count for table: %s: %s", task.TableNameTup, err) } return rowCount } @@ -950,7 +950,7 @@ func cleanImportState(state *ImportDataState, tasks []*ImportFileTask) { for _, task := range tasks { err := state.Clean(task.FilePath, task.TableNameTup) if err != nil { - utils.ErrExit("failed to clean import data state for table %q: %s", task.TableNameTup, err) + utils.ErrExit("failed to clean import data state for table: %q: %s", task.TableNameTup, err) } } @@ -958,7 +958,7 @@ func cleanImportState(state *ImportDataState, tasks []*ImportFileTask) { if utils.FileOrFolderExists(sqlldrDir) { err := os.RemoveAll(sqlldrDir) if err != nil { - utils.ErrExit("failed to remove sqlldr directory %q: %s", sqlldrDir, err) + utils.ErrExit("failed to remove sqlldr directory: %q: %s", sqlldrDir, err) } } @@ -1013,7 +1013,7 @@ func importFile(state *ImportDataState, task *ImportFileTask, updateProgressFn f log.Infof("Collect all interrupted/remaining splits.") pendingBatches, lastBatchNumber, lastOffset, fileFullySplit, err := state.Recover(task.FilePath, task.TableNameTup) if err != nil { - utils.ErrExit("recovering state for table %q: %s", task.TableNameTup, err) + utils.ErrExit("recovering state for table: %q: %s", task.TableNameTup, err) } for _, batch := range pendingBatches { submitBatch(batch, updateProgressFn, importBatchArgsProto) @@ -1031,12 +1031,12 @@ func splitFilesForTable(state *ImportDataState, filePath string, t sqlname.NameT reader, err := dataStore.Open(filePath) if err != nil { - utils.ErrExit("preparing reader for split generation on file %q: %v", filePath, err) + utils.ErrExit("preparing reader for split generation on file: %q: %v", filePath, err) } dataFile, err := datafile.NewDataFile(filePath, reader, dataFileDescriptor) if err != nil { - utils.ErrExit("open datafile %q: %v", filePath, err) + utils.ErrExit("open datafile: %q: %v", filePath, err) } defer dataFile.Close() @@ -1060,13 +1060,13 @@ func splitFilesForTable(state *ImportDataState, filePath string, t sqlname.NameT batchWriter = state.NewBatchWriter(filePath, t, batchNum) err := batchWriter.Init() if err != nil { - utils.ErrExit("initializing batch writer for table %q: %s", t, err) + utils.ErrExit("initializing batch writer for table: %q: %s", t, err) } // Write the header if necessary if header != "" && dataFileDescriptor.FileFormat == datafile.CSV { err = batchWriter.WriteHeader(header) if err != nil { - utils.ErrExit("writing header for table %q: %s", t, err) + utils.ErrExit("writing header for table: %q: %s", t, err) } } } @@ -1104,14 +1104,14 @@ func splitFilesForTable(state *ImportDataState, filePath string, t sqlname.NameT if tconf.TargetDBType == YUGABYTEDB { ybSpecificMsg = ", but should be strictly lower than the the rpc_max_message_size on YugabyteDB (default 267386880 bytes)" } - utils.ErrExit("record num=%d for table %q in file %s is larger than the max batch size %d bytes Max Batch size can be changed using env var MAX_BATCH_SIZE_BYTES%s", numLinesTaken, t.ForOutput(), filePath, tdb.MaxBatchSizeInBytes(), ybSpecificMsg) + utils.ErrExit("record of size %d larger than max batch size: record num=%d for table %q in file %s is larger than the max batch size %d bytes. Max Batch size can be changed using env var MAX_BATCH_SIZE_BYTES%s", currentBytesRead, numLinesTaken, t.ForOutput(), filePath, tdb.MaxBatchSizeInBytes(), ybSpecificMsg) } if line != "" { // can't use importBatchArgsProto.Columns as to use case insenstiive column names columnNames, _ := TableToColumnNames.Get(t) line, err = valueConverter.ConvertRow(t, columnNames, line) if err != nil { - utils.ErrExit("transforming line number=%d for table %q in file %s: %s", numLinesTaken, t.ForOutput(), filePath, err) + utils.ErrExit("transforming line number=%d for table: %q in file %s: %s", numLinesTaken, t.ForOutput(), filePath, err) } // Check if adding this record exceeds the max batch size @@ -1140,7 +1140,7 @@ func splitFilesForTable(state *ImportDataState, filePath string, t sqlname.NameT finalizeBatch(true, numLinesTaken, dataFile.GetBytesRead()) dataFile.ResetBytesRead(0) } else if readLineErr != nil { - utils.ErrExit("read line from data file %q: %s", filePath, readLineErr) + utils.ErrExit("read line from data file: %q: %s", filePath, readLineErr) } } @@ -1177,7 +1177,7 @@ func submitBatch(batch *Batch, updateProgressFn func(int64), importBatchArgsProt func importBatch(batch *Batch, importBatchArgsProto *tgtdb.ImportBatchArgs) { err := batch.MarkPending() if err != nil { - utils.ErrExit("marking batch %d as pending: %s", batch.Number, err) + utils.ErrExit("marking batch as pending: %d: %s", batch.Number, err) } log.Infof("Importing %q", batch.FilePath) @@ -1204,11 +1204,11 @@ func importBatch(batch *Batch, importBatchArgsProto *tgtdb.ImportBatchArgs) { } log.Infof("%q => %d rows affected", batch.FilePath, rowsAffected) if err != nil { - utils.ErrExit("import %q into %s: %s", batch.FilePath, batch.TableNameTup, err) + utils.ErrExit("import batch: %q into %s: %s", batch.FilePath, batch.TableNameTup, err) } err = batch.MarkDone() if err != nil { - utils.ErrExit("marking batch %q as done: %s", batch.FilePath, err) + utils.ErrExit("marking batch as done: %q: %s", batch.FilePath, err) } } @@ -1238,7 +1238,7 @@ func getTargetSchemaName(tableName string) string { if tconf.TargetDBType == POSTGRESQL { defaultSchema, noDefaultSchema := GetDefaultPGSchema(tconf.Schema, ",") if noDefaultSchema { - utils.ErrExit("no default schema for table %q ", tableName) + utils.ErrExit("no default schema for table: %q ", tableName) } return defaultSchema } @@ -1255,11 +1255,11 @@ func prepareTableToColumns(tasks []*ImportFileTask) { // File is either exported from debezium OR this is `import data file` case. reader, err := dataStore.Open(task.FilePath) if err != nil { - utils.ErrExit("datastore.Open %q: %v", task.FilePath, err) + utils.ErrExit("datastore.Open: %q: %v", task.FilePath, err) } df, err := datafile.NewDataFile(task.FilePath, reader, dataFileDescriptor) if err != nil { - utils.ErrExit("opening datafile %q: %v", task.FilePath, err) + utils.ErrExit("opening datafile: %q: %v", task.FilePath, err) } header := df.GetHeader() columns = strings.Split(header, dataFileDescriptor.Delimiter) @@ -1280,7 +1280,7 @@ func getDfdTableNameToExportedColumns(dataFileDescriptor *datafile.Descriptor) * for tableNameRaw, columnList := range dataFileDescriptor.TableNameToExportedColumns { nt, err := namereg.NameReg.LookupTableName(tableNameRaw) if err != nil { - utils.ErrExit("lookup table [%s] in name registry: %v", tableNameRaw, err) + utils.ErrExit("lookup table in name registry: %q: %v", tableNameRaw, err) } result.Put(nt, columnList) } diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 9af7e99f18..340d7ef057 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -185,7 +185,7 @@ func prepareImportFileTasks() []*ImportFileTask { for _, filePath := range filePaths { fileSize, err := dataStore.FileSize(filePath) if err != nil { - utils.ErrExit("calculating file size of %q in bytes: %v", filePath, err) + utils.ErrExit("calculating file size in bytes: %q: %v", filePath, err) } task := &ImportFileTask{ ID: i, @@ -247,12 +247,12 @@ func checkDataDirFlag() { } dataDirAbs, err := filepath.Abs(dataDir) if err != nil { - utils.ErrExit("unable to resolve absolute path for data-dir(%q): %v", dataDir, err) + utils.ErrExit("unable to resolve absolute path for data-dir: (%q): %v", dataDir, err) } exportDirAbs, err := filepath.Abs(exportDir) if err != nil { - utils.ErrExit("unable to resolve absolute path for export-dir(%q): %v", exportDir, err) + utils.ErrExit("unable to resolve absolute path for export-dir: (%q): %v", exportDir, err) } if strings.HasPrefix(dataDirAbs, exportDirAbs) { diff --git a/yb-voyager/cmd/importDataStatusCommand.go b/yb-voyager/cmd/importDataStatusCommand.go index aabf04ba37..62e4ddf72d 100644 --- a/yb-voyager/cmd/importDataStatusCommand.go +++ b/yb-voyager/cmd/importDataStatusCommand.go @@ -102,7 +102,7 @@ func runImportDataStatusCmd() error { reportFile := jsonfile.NewJsonFile[[]*tableMigStatusOutputRow](reportFilePath) err := reportFile.Create(&rows) if err != nil { - utils.ErrExit("creating into json file %s: %v", reportFilePath, err) + utils.ErrExit("creating into json file: %s: %v", reportFilePath, err) } fmt.Print(color.GreenString("Import data status report is written to %s\n", reportFilePath)) return nil diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index 250309618e..367ac0002a 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -307,7 +307,7 @@ func isYBDatabaseIsColocated(conn *pgx.Conn) bool { query := "SELECT yb_is_database_colocated();" err := conn.QueryRow(context.Background(), query).Scan(&isColocated) if err != nil { - utils.ErrExit("failed to check if Target DB '%s' is colocated or not: %v", tconf.DBName, err) + utils.ErrExit("failed to check if Target DB is colocated or not: %q: %v", tconf.DBName, err) } log.Infof("target DB '%s' colocoated='%t'", tconf.DBName, isColocated) return isColocated @@ -341,7 +341,7 @@ func dumpStatements(stmts []string, filePath string) { for i := 0; i < len(stmts); i++ { _, err = file.WriteString(stmts[i] + "\n\n") if err != nil { - utils.ErrExit("failed writing in file %s: %v", filePath, err) + utils.ErrExit("failed writing in file: %s: %v", filePath, err) } } @@ -377,7 +377,7 @@ func refreshMViews(conn *pgx.Conn) { query := fmt.Sprintf("REFRESH MATERIALIZED VIEW %s", mViewName) _, err := conn.Exec(context.Background(), query) if err != nil && !strings.Contains(strings.ToLower(err.Error()), "has not been populated") { - utils.ErrExit("error in refreshing the materialized view %s: %v", mViewName, err) + utils.ErrExit("error in refreshing the materialized view: %s: %v", mViewName, err) } } log.Infof("Checking if mviews are refreshed or not - %v", mViewNames) @@ -386,7 +386,7 @@ func refreshMViews(conn *pgx.Conn) { query := fmt.Sprintf("SELECT * from %s LIMIT 1;", mViewName) rows, err := conn.Query(context.Background(), query) if err != nil { - utils.ErrExit("error in checking whether mview %s is refreshed or not: %v", mViewName, err) + utils.ErrExit("error in checking whether mview is refreshed or not: %q: %v", mViewName, err) } if !rows.Next() { mviewsNotRefreshed = append(mviewsNotRefreshed, mViewName) @@ -448,7 +448,7 @@ func createTargetSchemas(conn *pgx.Conn) { utils.PrintAndLog("dropping schema '%s' in target database", targetSchema) _, err := conn.Exec(context.Background(), dropSchemaQuery) if err != nil { - utils.ErrExit("Failed to drop schema %q: %s", targetSchema, err) + utils.ErrExit("Failed to drop schema: %q: %s", targetSchema, err) } } else { utils.PrintAndLog("schema '%s' already present in target database, continuing with it..\n", targetSchema) @@ -465,7 +465,7 @@ func createTargetSchemas(conn *pgx.Conn) { utils.PrintAndLog("creating schema '%s' in target database...", tconf.Schema) _, err := conn.Exec(context.Background(), createSchemaQuery) if err != nil { - utils.ErrExit("Failed to create %q schema in the target DB: %s", tconf.Schema, err) + utils.ErrExit("Failed to create schema in the target DB: %q: %s", tconf.Schema, err) } } @@ -485,7 +485,7 @@ func checkIfTargetSchemaExists(conn *pgx.Conn, targetSchema string) bool { if err != nil && (strings.Contains(err.Error(), "no rows in result set") && fetchedSchema == "") { return false } else if err != nil { - utils.ErrExit("Failed to check if schema %q exists: %s", targetSchema, err) + utils.ErrExit("Failed to check if schema exists: %q: %s", targetSchema, err) } return fetchedSchema == targetSchema diff --git a/yb-voyager/cmd/importSchemaYugabyteDB.go b/yb-voyager/cmd/importSchemaYugabyteDB.go index 79720d1477..27fae75f2e 100644 --- a/yb-voyager/cmd/importSchemaYugabyteDB.go +++ b/yb-voyager/cmd/importSchemaYugabyteDB.go @@ -468,15 +468,15 @@ func setTargetSchema(conn *pgx.Conn) { var cntSchemaName int if err := conn.QueryRow(context.Background(), checkSchemaExistsQuery).Scan(&cntSchemaName); err != nil { - utils.ErrExit("run query %q on target %q to check schema exists: %s", checkSchemaExistsQuery, tconf.Host, err) + utils.ErrExit("run query: %q on target %q to check schema exists: %s", checkSchemaExistsQuery, tconf.Host, err) } else if cntSchemaName == 0 { - utils.ErrExit("schema '%s' does not exist in target", tconf.Schema) + utils.ErrExit("schema does not exist in target: %q", tconf.Schema) } setSchemaQuery := fmt.Sprintf("SET SCHEMA '%s'", tconf.Schema) _, err := conn.Exec(context.Background(), setSchemaQuery) if err != nil { - utils.ErrExit("run query %q on target %q: %s", setSchemaQuery, tconf.Host, err) + utils.ErrExit("run query: %q on target %q: %s", setSchemaQuery, tconf.Host, err) } } diff --git a/yb-voyager/cmd/root.go b/yb-voyager/cmd/root.go index c9e7f87cca..a90ddcd69e 100644 --- a/yb-voyager/cmd/root.go +++ b/yb-voyager/cmd/root.go @@ -319,7 +319,7 @@ func validateExportDirFlag() { utils.ErrExit(`ERROR: required flag "export-dir" not set`) } if !utils.FileOrFolderExists(exportDir) { - utils.ErrExit("export-dir %q doesn't exists.\n", exportDir) + utils.ErrExit("export-dir doesn't exist: %q\n", exportDir) } else { if exportDir == "." { fmt.Println("Note: Using current directory as export-dir") @@ -327,7 +327,7 @@ func validateExportDirFlag() { var err error exportDir, err = filepath.Abs(exportDir) if err != nil { - utils.ErrExit("Failed to get absolute path for export-dir %q: %v\n", exportDir, err) + utils.ErrExit("Failed to get absolute path for export-dir: %q: %v\n", exportDir, err) } exportDir = filepath.Clean(exportDir) } diff --git a/yb-voyager/src/datastore/azDatastore.go b/yb-voyager/src/datastore/azDatastore.go index dbd06e7791..a2c1b432f5 100644 --- a/yb-voyager/src/datastore/azDatastore.go +++ b/yb-voyager/src/datastore/azDatastore.go @@ -35,7 +35,7 @@ type AzDataStore struct { func NewAzDataStore(dataDir string) *AzDataStore { url, err := url.Parse(dataDir) if err != nil { - utils.ErrExit("invalid azure resource URL %v", dataDir) + utils.ErrExit("invalid azure resource URL: %v", dataDir) } return &AzDataStore{url: url} } @@ -80,7 +80,7 @@ func (ds *AzDataStore) Open(objectPath string) (io.ReadCloser, error) { // if objectPath is hidden underneath a symlink for az blobs... objectPath, err := os.Readlink(objectPath) if err != nil { - utils.ErrExit("unable to resolve symlink %v to gcs resource: %w", objectPath, err) + utils.ErrExit("unable to resolve symlink: %v to gcs resource: %w", objectPath, err) } return az.NewObjectReader(objectPath) } diff --git a/yb-voyager/src/datastore/gcsDatastore.go b/yb-voyager/src/datastore/gcsDatastore.go index 312843e756..5e6bdbe7ef 100644 --- a/yb-voyager/src/datastore/gcsDatastore.go +++ b/yb-voyager/src/datastore/gcsDatastore.go @@ -17,12 +17,12 @@ limitations under the License. package datastore import ( + "fmt" "io" "net/url" "os" "regexp" "strings" - "fmt" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/gcs" @@ -36,7 +36,7 @@ type GCSDataStore struct { func NewGCSDataStore(resourceName string) *GCSDataStore { url, err := url.Parse(resourceName) if err != nil { - utils.ErrExit("invalid gcs resource URL %v", resourceName) + utils.ErrExit("invalid gcs resource URL: %v", resourceName) } return &GCSDataStore{url: url, bucketName: url.Host} } @@ -48,13 +48,13 @@ func (ds *GCSDataStore) Glob(pattern string) ([]string, error) { return nil, fmt.Errorf("listing all objects of %q: %w", pattern, err) } pattern = strings.Replace(pattern, "*", ".*", -1) - pattern = ds.url.String() + "/" + pattern + pattern = ds.url.String() + "/" + pattern re := regexp.MustCompile(pattern) var resultSet []string for _, objectName := range objectNames { objectName = ds.url.String() + "/" + objectName if re.MatchString(objectName) { - resultSet = append(resultSet, objectName) + resultSet = append(resultSet, objectName) } } return resultSet, nil @@ -68,7 +68,7 @@ func (ds *GCSDataStore) AbsolutePath(filePath string) (string, error) { func (ds *GCSDataStore) FileSize(filePath string) (int64, error) { objAttrs, err := gcs.GetObjAttrs(filePath) if err != nil { - return 0, fmt.Errorf("get attributes of %q: %w",filePath, err) + return 0, fmt.Errorf("get attributes of %q: %w", filePath, err) } return objAttrs.Size, nil } @@ -80,7 +80,7 @@ func (ds *GCSDataStore) Open(resourceName string) (io.ReadCloser, error) { // if resourceName is hidden underneath a symlink for gcs objects... objectPath, err := os.Readlink(resourceName) if err != nil { - utils.ErrExit("unable to resolve symlink %v to gcs resource: %w", resourceName, err) + utils.ErrExit("unable to resolve symlink: %v to gcs resource: %w", resourceName, err) } return gcs.NewObjectReader(objectPath) } diff --git a/yb-voyager/src/datastore/localDatastore.go b/yb-voyager/src/datastore/localDatastore.go index 723ce454d7..55ba4aad44 100644 --- a/yb-voyager/src/datastore/localDatastore.go +++ b/yb-voyager/src/datastore/localDatastore.go @@ -31,7 +31,7 @@ type LocalDataStore struct { func NewLocalDataStore(dataDir string) *LocalDataStore { dataDir, err := filepath.Abs(dataDir) if err != nil { - utils.ErrExit("failed to get absolute path of directory %q: %s", dataDir, err) + utils.ErrExit("failed to get absolute path of directory: %q: %s", dataDir, err) } dataDir = filepath.Clean(dataDir) return &LocalDataStore{dataDir: dataDir} diff --git a/yb-voyager/src/datastore/s3Datastore.go b/yb-voyager/src/datastore/s3Datastore.go index 60fb280539..909e3a34e7 100644 --- a/yb-voyager/src/datastore/s3Datastore.go +++ b/yb-voyager/src/datastore/s3Datastore.go @@ -17,10 +17,10 @@ limitations under the License. package datastore import ( + "fmt" "io" "net/url" "os" - "fmt" "regexp" "strings" @@ -36,7 +36,7 @@ type S3DataStore struct { func NewS3DataStore(resourceName string) *S3DataStore { url, err := url.Parse(resourceName) if err != nil { - utils.ErrExit("invalid s3 resource URL %v", resourceName) + utils.ErrExit("invalid s3 resource URL: %v", resourceName) } return &S3DataStore{url: url, bucketName: url.Host} } @@ -80,7 +80,7 @@ func (ds *S3DataStore) Open(resourceName string) (io.ReadCloser, error) { // if resourceName is hidden underneath a symlink for s3 objects... objectPath, err := os.Readlink(resourceName) if err != nil { - utils.ErrExit("unable to resolve symlink %v to s3 resource: %w", resourceName, err) + utils.ErrExit("unable to resolve symlink to s3 resource: %v: %w", resourceName, err) } return s3.NewObjectReader(objectPath) } diff --git a/yb-voyager/src/dbzm/status.go b/yb-voyager/src/dbzm/status.go index d176272498..5e0c0d2838 100644 --- a/yb-voyager/src/dbzm/status.go +++ b/yb-voyager/src/dbzm/status.go @@ -82,7 +82,7 @@ func IsMigrationInStreamingMode(exportDir string) bool { statusFilePath := filepath.Join(exportDir, "data", "export_status.json") status, err := ReadExportStatus(statusFilePath) if err != nil { - utils.ErrExit("Failed to read export status file %s: %v", statusFilePath, err) + utils.ErrExit("Failed to read export status file: %s: %v", statusFilePath, err) } return status != nil && status.Mode == MODE_STREAMING } diff --git a/yb-voyager/src/srcdb/common.go b/yb-voyager/src/srcdb/common.go index 0d20ead96a..2035d537a7 100644 --- a/yb-voyager/src/srcdb/common.go +++ b/yb-voyager/src/srcdb/common.go @@ -38,7 +38,7 @@ func getExportedDataFileList(tablesMetadata map[string]*utils.TableProgressMetad targetTableName := strings.TrimSuffix(filepath.Base(tableMetadata.FinalFilePath), "_data.sql") table, err := namereg.NameReg.LookupTableName(targetTableName) if err != nil { - utils.ErrExit("error while looking up table name %q: %v", targetTableName, err) + utils.ErrExit("error while looking up table name: %q: %v", targetTableName, err) } if !utils.FileOrFolderExists(tableMetadata.FinalFilePath) { // This can happen in case of nested tables in Oracle. diff --git a/yb-voyager/src/srcdb/mysql.go b/yb-voyager/src/srcdb/mysql.go index a30dce839e..3090682d18 100644 --- a/yb-voyager/src/srcdb/mysql.go +++ b/yb-voyager/src/srcdb/mysql.go @@ -94,7 +94,7 @@ func (ms *MySQL) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 { log.Infof("Querying '%s' approx row count of table %q", query, tableName.String()) err := ms.db.QueryRow(query).Scan(&approxRowCount) if err != nil { - utils.ErrExit("Failed to query %q for approx row count of %q: %s", query, tableName.String(), err) + utils.ErrExit("Failed to query for approx row count of table: %q: %q %s", tableName.String(), query, err) } log.Infof("Table %q has approx %v rows.", tableName.String(), approxRowCount) @@ -110,7 +110,7 @@ func (ms *MySQL) GetVersion() string { query := "SELECT VERSION()" err := ms.db.QueryRow(query).Scan(&version) if err != nil { - utils.ErrExit("run query %q on source: %s", query, err) + utils.ErrExit("run query: %q on source: %s", query, err) } ms.source.DBVersion = version return version @@ -388,7 +388,7 @@ func (ms *MySQL) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[strin var columnName string rows, err := ms.db.Query(query) if err != nil { - utils.ErrExit("Failed to query %q for auto increment column of %q: %s", query, table.String(), err) + utils.ErrExit("Failed to query for auto increment column: query:%q table: %q: %s", query, table.String(), err) } defer func() { closeErr := rows.Close() @@ -399,7 +399,7 @@ func (ms *MySQL) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[strin if rows.Next() { err = rows.Scan(&columnName) if err != nil { - utils.ErrExit("Failed to scan %q for auto increment column of %q: %s", query, table.String(), err) + utils.ErrExit("Failed to scan for auto increment column: query: %q table: %q: %s", query, table.String(), err) } qualifiedColumeName := fmt.Sprintf("%s.%s", table.AsQualifiedCatalogName(), columnName) // sequence name as per PG naming convention for bigserial datatype's sequence @@ -408,7 +408,7 @@ func (ms *MySQL) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[strin } err = rows.Close() if err != nil { - utils.ErrExit("close rows for table %s query %q: %s", table.String(), query, err) + utils.ErrExit("close rows for table: %s query %q: %s", table.String(), query, err) } } return columnToSequenceMap diff --git a/yb-voyager/src/srcdb/ora2pg.go b/yb-voyager/src/srcdb/ora2pg.go index 98d8272d68..bc299a9c5d 100644 --- a/yb-voyager/src/srcdb/ora2pg.go +++ b/yb-voyager/src/srcdb/ora2pg.go @@ -113,7 +113,7 @@ func populateOra2pgConfigFile(configFilePath string, conf *Ora2pgConfig) { err = os.WriteFile(configFilePath, output.Bytes(), 0644) if err != nil { - utils.ErrExit("unable to update config file %q: %v\n", configFilePath, err) + utils.ErrExit("unable to update config file: %q: %v\n", configFilePath, err) } } diff --git a/yb-voyager/src/srcdb/ora2pg_export_data.go b/yb-voyager/src/srcdb/ora2pg_export_data.go index 4357b4fa9e..d0f6dfe6e2 100644 --- a/yb-voyager/src/srcdb/ora2pg_export_data.go +++ b/yb-voyager/src/srcdb/ora2pg_export_data.go @@ -118,7 +118,7 @@ func getIdentityColumnSequences(exportDir string) []string { filePath := filepath.Join(exportDir, "data", "postdata.sql") bytes, err := os.ReadFile(filePath) if err != nil { - utils.ErrExit("unable to read file %q: %v\n", filePath, err) + utils.ErrExit("unable to read file: %q: %v\n", filePath, err) } lines := strings.Split(string(bytes), "\n") @@ -137,7 +137,7 @@ func replaceAllIdentityColumns(exportDir string, sourceTargetIdentitySequenceNam filePath := filepath.Join(exportDir, "data", "postdata.sql") bytes, err := os.ReadFile(filePath) if err != nil { - utils.ErrExit("unable to read file %q: %v\n", filePath, err) + utils.ErrExit("unable to read file: %q: %v\n", filePath, err) } lines := strings.Split(string(bytes), "\n") @@ -160,7 +160,7 @@ func replaceAllIdentityColumns(exportDir string, sourceTargetIdentitySequenceNam err = os.WriteFile(filePath, bytesToWrite, 0644) if err != nil { - utils.ErrExit("unable to write file %q: %v\n", filePath, err) + utils.ErrExit("unable to write file: %q: %v\n", filePath, err) } } @@ -180,7 +180,7 @@ func renameDataFilesForReservedWords(tablesProgressMetadata map[string]*utils.Ta log.Infof("Renaming %q -> %q", oldFilePath, newFilePath) err := os.Rename(oldFilePath, newFilePath) if err != nil { - utils.ErrExit("renaming data file for table %q after data export: %v", tblNameQuoted, err) + utils.ErrExit("renaming data file for table after data export: %q: %v", tblNameQuoted, err) } tableProgressMetadata.FinalFilePath = newFilePath } else { @@ -214,7 +214,7 @@ func getOra2pgExportedColumnsListForTable(exportDir, tableName, filePath string) return false // stop reading file }) if err != nil { - utils.ErrExit("error in reading file %q: %v", filePath, err) + utils.ErrExit("error in reading file: %q: %v", filePath, err) } log.Infof("columns list for table %s: %v", tableName, columnsList) return columnsList diff --git a/yb-voyager/src/srcdb/oracle.go b/yb-voyager/src/srcdb/oracle.go index 5527963ec7..32de02bfa7 100644 --- a/yb-voyager/src/srcdb/oracle.go +++ b/yb-voyager/src/srcdb/oracle.go @@ -73,7 +73,7 @@ func (ora *Oracle) CheckSchemaExists() bool { if err == sql.ErrNoRows { return false } else if err != nil { - utils.ErrExit("error in querying source database for schema %q: %v\n", schemaName, err) + utils.ErrExit("error in querying source database for schema: %q: %v\n", schemaName, err) } return true } @@ -101,7 +101,7 @@ func (ora *Oracle) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 { log.Infof("Querying '%s' approx row count of table %q", query, tableName.String()) err := ora.db.QueryRow(query).Scan(&approxRowCount) if err != nil { - utils.ErrExit("Failed to query %q for approx row count of %q: %s", query, tableName.String(), err) + utils.ErrExit("Failed to query: %q for approx row count of %q: %s", query, tableName.String(), err) } log.Infof("Table %q has approx %v rows.", tableName.String(), approxRowCount) @@ -118,7 +118,7 @@ func (ora *Oracle) GetVersion() string { // query sample output: Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production err := ora.db.QueryRow(query).Scan(&version) if err != nil { - utils.ErrExit("run query %q on source: %s", query, err) + utils.ErrExit("run query: %q on source: %s", query, err) } ora.source.DBVersion = version return version @@ -325,13 +325,13 @@ func (ora *Oracle) FilterUnsupportedTables(migrationUUID uuid.UUID, tableList [] log.Infof("query for queue tables: %q\n", query) rows, err := ora.db.Query(query) if err != nil { - utils.ErrExit("failed to query %q for filtering unsupported queue tables: %v", query, err) + utils.ErrExit("failed to query for filtering unsupported queue tables:%q: %v", query, err) } for rows.Next() { var tableName string err := rows.Scan(&tableName) if err != nil { - utils.ErrExit("failed to scan tableName from output of query %q: %v", query, err) + utils.ErrExit("failed to scan tableName from output of query: %q: %v", query, err) } tableName = fmt.Sprintf(`"%s"`, tableName) tableSrcName := sqlname.NewSourceName(ora.source.Schema, tableName) @@ -394,7 +394,7 @@ func (ora *Oracle) IsNestedTable(tableName sqlname.NameTuple) bool { isNestedTable := 0 err := ora.db.QueryRow(query).Scan(&isNestedTable) if err != nil && err != sql.ErrNoRows { - utils.ErrExit("error in query to check if table %v is a nested table: %v", tableName, err) + utils.ErrExit("check if table is a nested table: %v: %v", tableName, err) } return isNestedTable == 1 } @@ -407,7 +407,7 @@ func (ora *Oracle) IsParentOfNestedTable(tableName sqlname.NameTuple) bool { isParentNestedTable := 0 err := ora.db.QueryRow(query).Scan(&isParentNestedTable) if err != nil && err != sql.ErrNoRows { - utils.ErrExit("error in query to check if table %v is parent of nested table: %v", tableName, err) + utils.ErrExit("check if table is parent of nested table: %v: %v", tableName, err) } return isParentNestedTable == 1 } @@ -420,7 +420,7 @@ func (ora *Oracle) GetTargetIdentityColumnSequenceName(sequenceName string) stri if err == sql.ErrNoRows { return "" } else if err != nil { - utils.ErrExit("failed to query %q for finding identity sequence table and column: %v", query, err) + utils.ErrExit("failed to query for finding identity sequence table and column: %q: %v", query, err) } return fmt.Sprintf("%s_%s_seq", tableName, columnName) @@ -442,7 +442,7 @@ func (ora *Oracle) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[str query := fmt.Sprintf("SELECT column_name FROM all_tab_identity_cols WHERE owner = '%s' AND table_name = '%s'", sname, tname) rows, err := ora.db.Query(query) if err != nil { - utils.ErrExit("failed to query %q for finding identity column: %v", query, err) + utils.ErrExit("failed to query for finding identity column: %q: %v", query, err) } defer func() { closeErr := rows.Close() @@ -454,14 +454,14 @@ func (ora *Oracle) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[str var columnName string err := rows.Scan(&columnName) if err != nil { - utils.ErrExit("failed to scan columnName from output of query %q: %v", query, err) + utils.ErrExit("failed to scan columnName from output of query: %q: %v", query, err) } qualifiedColumnName := fmt.Sprintf("%s.%s", table.AsQualifiedCatalogName(), columnName) columnToSequenceMap[qualifiedColumnName] = fmt.Sprintf("%s_%s_seq", tname, columnName) } err = rows.Close() if err != nil { - utils.ErrExit("close rows for table %s query %q: %s", table.String(), query, err) + utils.ErrExit("close rows for table: %s query %q: %s", table.String(), query, err) } } diff --git a/yb-voyager/src/srcdb/pg_dump_export_data.go b/yb-voyager/src/srcdb/pg_dump_export_data.go index a41cf186a4..d68871409d 100644 --- a/yb-voyager/src/srcdb/pg_dump_export_data.go +++ b/yb-voyager/src/srcdb/pg_dump_export_data.go @@ -108,7 +108,7 @@ func parseAndCreateTocTextFile(dataDirPath string) { parseTocFileCommand := exec.Command("strings", tocFilePath) cmdOutput, err := parseTocFileCommand.CombinedOutput() if err != nil { - utils.ErrExit("parsing tocfile %q: %v", tocFilePath, err) + utils.ErrExit("parsing tocfile: %q: %v", tocFilePath, err) } //Put the data into a toc.txt file @@ -146,7 +146,7 @@ func renameDataFiles(tablesProgressMetadata map[string]*utils.TableProgressMetad log.Infof("Renaming %q -> %q", oldFilePath, newFilePath) err := os.Rename(oldFilePath, newFilePath) if err != nil { - utils.ErrExit("renaming data file for table %q after data export: %v", tableProgressMetadata.TableName, err) + utils.ErrExit("renaming data file: for table %q after data export: %v", tableProgressMetadata.TableName, err) } } else { log.Infof("File %q to rename doesn't exists!", oldFilePath) diff --git a/yb-voyager/src/srcdb/pg_dump_extract_schema.go b/yb-voyager/src/srcdb/pg_dump_extract_schema.go index 71c09c433a..20e5158839 100644 --- a/yb-voyager/src/srcdb/pg_dump_extract_schema.go +++ b/yb-voyager/src/srcdb/pg_dump_extract_schema.go @@ -70,7 +70,7 @@ func pgdumpExtractSchema(source *Source, connectionUri string, exportDir string, func readSchemaFile(path string) []string { file, err := os.Open(path) if err != nil { - utils.ErrExit("error in opening schema file %s: %v", path, err) + utils.ErrExit("error in opening schema file: %s: %v", path, err) } defer file.Close() var lines []string @@ -83,7 +83,7 @@ func readSchemaFile(path string) []string { } if scanner.Err() != nil { - utils.ErrExit("error in reading schema file %s: %v", path, scanner.Err()) + utils.ErrExit("error in reading schema file: %s: %v", path, scanner.Err()) } return lines diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 58c4d142a2..48406e6abb 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -160,7 +160,7 @@ func (pg *PostgreSQL) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 log.Infof("Querying '%s' approx row count of table %q", query, tableName.String()) err := pg.db.QueryRow(query).Scan(&approxRowCount) if err != nil { - utils.ErrExit("Failed to query %q for approx row count of %q: %s", query, tableName.String(), err) + utils.ErrExit("Failed to query for approx row count of table: %q: %q: %s", tableName.String(), query, err) } log.Infof("Table %q has approx %v rows.", tableName.String(), approxRowCount) @@ -195,7 +195,7 @@ func (pg *PostgreSQL) checkSchemasExists() []string { WHERE nspname IN (%s);`, querySchemaList) rows, err := pg.db.Query(chkSchemaExistsQuery) if err != nil { - utils.ErrExit("error in querying(%q) source database for checking mentioned schema(s) present or not: %v\n", chkSchemaExistsQuery, err) + utils.ErrExit("error in querying source database for checking mentioned schema(s) present or not: %q: %v\n", chkSchemaExistsQuery, err) } defer func() { closeErr := rows.Close() @@ -216,7 +216,7 @@ func (pg *PostgreSQL) checkSchemasExists() []string { schemaNotPresent := utils.SetDifference(trimmedSchemaList, listOfSchemaPresent) if len(schemaNotPresent) > 0 { - utils.ErrExit("Following schemas are not present in source database %v, please provide a valid schema list.\n", schemaNotPresent) + utils.ErrExit("Following schemas are not present in source database: %v, please provide a valid schema list.\n", schemaNotPresent) } return trimmedSchemaList } @@ -281,7 +281,7 @@ func (pg *PostgreSQL) GetAllTableNames() []*sqlname.SourceName { rows, err := pg.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for table names: %v\n", query, err) + utils.ErrExit("error in querying source database for table names: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -467,7 +467,7 @@ func (pg *PostgreSQL) GetAllSequences() []string { query := fmt.Sprintf(`SELECT sequence_schema, sequence_name FROM information_schema.sequences where sequence_schema IN (%s);`, querySchemaList) rows, err := pg.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for sequence names: %v\n", query, err) + utils.ErrExit("error in querying source database for sequence names: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -593,7 +593,7 @@ func (pg *PostgreSQL) FilterEmptyTables(tableList []sqlname.NameTuple) ([]sqlnam if err == sql.ErrNoRows { empty = true } else { - utils.ErrExit("error in querying table %v: %v", tableName, err) + utils.ErrExit("error in querying table LIMIT 1: %v: %v", tableName, err) } } if !empty { @@ -686,7 +686,7 @@ func (pg *PostgreSQL) ParentTableOfPartition(table sqlname.NameTuple) string { err := pg.db.QueryRow(query).Scan(&parentTable) if err != sql.ErrNoRows && err != nil { - utils.ErrExit("Error in query=%s for parent tablename of table=%s: %v", query, table, err) + utils.ErrExit("Error in query: %s for parent tablename of table=%s: %v", query, table, err) } return parentTable @@ -704,7 +704,7 @@ func (pg *PostgreSQL) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[ rows, err := pg.db.Query(query) if err != nil { log.Infof("Query to find column to sequence mapping: %s", query) - utils.ErrExit("Error in querying for sequences in table=%s: %v", table, err) + utils.ErrExit("Error in querying for sequences in table: %s: %v", table, err) } defer func() { closeErr := rows.Close() @@ -715,7 +715,7 @@ func (pg *PostgreSQL) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[ for rows.Next() { err := rows.Scan(&columeName, &sequenceName, &schemaName) if err != nil { - utils.ErrExit("Error in scanning for sequences in table=%s: %v", table, err) + utils.ErrExit("Error in scanning for sequences in table: %s: %v", table, err) } qualifiedColumnName := fmt.Sprintf("%s.%s", table.AsQualifiedCatalogName(), columeName) // quoting sequence name as it can be case sensitive - required during import data restore sequences @@ -723,7 +723,7 @@ func (pg *PostgreSQL) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[ } err = rows.Close() if err != nil { - utils.ErrExit("close rows for table %s query %q: %s", table.String(), query, err) + utils.ErrExit("close rows for table: %s query %q: %s", table.String(), query, err) } } @@ -785,7 +785,7 @@ WHERE parent.relname='%s' AND nmsp_parent.nspname = '%s' `, tname, sname) rows, err := pg.db.Query(query) if err != nil { log.Errorf("failed to list partitions of table %s: query = [ %s ], error = %s", tableName, query, err) - utils.ErrExit("failed to find the partitions for table %s: %v", tableName, err) + utils.ErrExit("failed to find the partitions for table: %s: %v", tableName, err) } defer func() { closeErr := rows.Close() @@ -797,12 +797,12 @@ WHERE parent.relname='%s' AND nmsp_parent.nspname = '%s' `, tname, sname) var childSchema, childTable string err := rows.Scan(&childSchema, &childTable) if err != nil { - utils.ErrExit("Error in scanning for child partitions of table=%s: %v", tableName, err) + utils.ErrExit("Error in scanning for child partitions of table: %s: %v", tableName, err) } partitions = append(partitions, fmt.Sprintf(`%s.%s`, childSchema, childTable)) } if rows.Err() != nil { - utils.ErrExit("Error in scanning for child partitions of table=%s: %v", tableName, rows.Err()) + utils.ErrExit("Error in scanning for child partitions of table: %s: %v", tableName, rows.Err()) } return partitions } @@ -1433,7 +1433,7 @@ func (pg *PostgreSQL) checkWalLevel() (msg string) { var walLevel string err := pg.db.QueryRow(query).Scan(&walLevel) if err != nil { - utils.ErrExit("error in querying(%q) source database for wal_level: %v\n", query, err) + utils.ErrExit("error in querying source database for wal_level: %q %v\n", query, err) } if walLevel != "logical" { msg = fmt.Sprintf("\n%s Current wal_level: %s; Required wal_level: logical", color.RedString("ERROR"), walLevel) diff --git a/yb-voyager/src/srcdb/srcdb.go b/yb-voyager/src/srcdb/srcdb.go index df707dcdab..aac963ff6e 100644 --- a/yb-voyager/src/srcdb/srcdb.go +++ b/yb-voyager/src/srcdb/srcdb.go @@ -84,7 +84,7 @@ func IsTableEmpty(db *sql.DB, query string) bool { return true } if err != nil { - utils.ErrExit("Failed to query %q to check table is empty: %s", query, err) + utils.ErrExit("Failed to query: %q to check table is empty: %s", query, err) } return false } diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index 2a90f8f052..96daa1307b 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -90,7 +90,7 @@ func (yb *YugabyteDB) GetTableApproxRowCount(tableName sqlname.NameTuple) int64 log.Infof("Querying '%s' approx row count of table %q", query, tableName.String()) err := yb.db.QueryRow(query).Scan(&approxRowCount) if err != nil { - utils.ErrExit("Failed to query %q for approx row count of %q: %s", query, tableName.String(), err) + utils.ErrExit("Failed to query: %q for approx row count of %q: %s", query, tableName.String(), err) } log.Infof("Table %q has approx %v rows.", tableName.String(), approxRowCount) @@ -106,7 +106,7 @@ func (yb *YugabyteDB) GetVersion() string { query := "SELECT setting from pg_settings where name = 'server_version'" err := yb.db.QueryRow(query).Scan(&version) if err != nil { - utils.ErrExit("run query %q on source: %s", query, err) + utils.ErrExit("run query: %q on source: %s", query, err) } yb.source.DBVersion = version return version @@ -131,7 +131,7 @@ func (yb *YugabyteDB) checkSchemasExists() []string { FROM information_schema.schemata where schema_name IN (%s);`, querySchemaList) rows, err := yb.db.Query(chkSchemaExistsQuery) if err != nil { - utils.ErrExit("error in querying(%q) source database for checking mentioned schema(s) present or not: %v\n", chkSchemaExistsQuery, err) + utils.ErrExit("error in querying source database for checking mentioned schema(s) present or not: %q: %v\n", chkSchemaExistsQuery, err) } var listOfSchemaPresent []string var tableSchemaName string @@ -152,7 +152,7 @@ func (yb *YugabyteDB) checkSchemasExists() []string { schemaNotPresent := utils.SetDifference(trimmedList, listOfSchemaPresent) if len(schemaNotPresent) > 0 { - utils.ErrExit("Following schemas are not present in source database %v, please provide a valid schema list.\n", schemaNotPresent) + utils.ErrExit("Following schemas are not present in source database: %v, please provide a valid schema list.\n", schemaNotPresent) } return trimmedList } @@ -217,7 +217,7 @@ func (yb *YugabyteDB) GetAllTableNames() []*sqlname.SourceName { rows, err := yb.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) YB database for table names: %v\n", query, err) + utils.ErrExit("error in querying YB database for table names: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -347,7 +347,7 @@ func (yb *YugabyteDB) GetAllSequences() []string { query := fmt.Sprintf(`SELECT sequence_name FROM information_schema.sequences where sequence_schema IN (%s);`, querySchemaList) rows, err := yb.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for sequence names: %v\n", query, err) + utils.ErrExit("error in querying source database for sequence names: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -434,7 +434,7 @@ func (yb *YugabyteDB) getAllUserDefinedTypesInSchema(schemaName string) []string );`, schemaName, schemaName, schemaName) rows, err := yb.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for enum types: %v\n", query, err) + utils.ErrExit("error in querying source database for enum types: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -461,7 +461,7 @@ func (yb *YugabyteDB) getTypesOfAllArraysInATable(schemaName, tableName string) AND data_type = 'ARRAY';`, schemaName, tableName) rows, err := yb.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for array types: %v\n", query, err) + utils.ErrExit("error in querying source database for array types: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -541,7 +541,7 @@ func (yb *YugabyteDB) FilterEmptyTables(tableList []sqlname.NameTuple) ([]sqlnam if err == sql.ErrNoRows { empty = true } else { - utils.ErrExit("error in querying table %v: %v", tableName, err) + utils.ErrExit("error in querying table: %v: %v", tableName, err) } } if !empty { @@ -602,7 +602,7 @@ func (yb *YugabyteDB) filterUnsupportedUserDefinedDatatypes(tableName sqlname.Na a.attnum;`, tname, sname) rows, err := yb.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for user defined columns: %v\n", query, err) + utils.ErrExit("error in querying source database for user defined columns: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() @@ -670,7 +670,7 @@ func (yb *YugabyteDB) ParentTableOfPartition(table sqlname.NameTuple) string { err := yb.db.QueryRow(query).Scan(&parentTable) if err != sql.ErrNoRows && err != nil { - utils.ErrExit("Error in query=%s for parent tablename of table=%s: %v", query, table, err) + utils.ErrExit("Error in query for parent tablename of table: %q: %s: %v", query, table, err) } return parentTable @@ -688,7 +688,7 @@ func (yb *YugabyteDB) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[ rows, err := yb.db.Query(query) if err != nil { log.Infof("Query to find column to sequence mapping: %s", query) - utils.ErrExit("Error in querying for sequences in table=%s: %v", table, err) + utils.ErrExit("Error in querying for sequences in table: %s: %v", table, err) } defer func() { closeErr := rows.Close() @@ -699,7 +699,7 @@ func (yb *YugabyteDB) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[ for rows.Next() { err := rows.Scan(&columeName, &sequenceName, &schemaName) if err != nil { - utils.ErrExit("Error in scanning for sequences in table=%s: %v", table, err) + utils.ErrExit("Error in scanning for sequences in table: %s: %v", table, err) } qualifiedColumnName := fmt.Sprintf("%s.%s", table.AsQualifiedCatalogName(), columeName) // quoting sequence name as it can be case sensitive - required during import data restore sequences @@ -707,7 +707,7 @@ func (yb *YugabyteDB) GetColumnToSequenceMap(tableList []sqlname.NameTuple) map[ } err = rows.Close() if err != nil { - utils.ErrExit("close rows for table %s query %q: %s", table.String(), query, err) + utils.ErrExit("close rows for table: %s query: %q: %s", table.String(), query, err) } } @@ -720,7 +720,7 @@ func (yb *YugabyteDB) GetServers() []string { YB_SERVERS_QUERY := "SELECT host FROM yb_servers()" rows, err := yb.db.Query(YB_SERVERS_QUERY) if err != nil { - utils.ErrExit("error in querying(%q) source database for yb_servers: %v\n", YB_SERVERS_QUERY, err) + utils.ErrExit("error in querying source database for yb_servers: %q: %v\n", YB_SERVERS_QUERY, err) } defer func() { closeErr := rows.Close() @@ -756,7 +756,7 @@ WHERE parent.relname='%s' AND nmsp_parent.nspname = '%s' `, tname, sname) rows, err := yb.db.Query(query) if err != nil { log.Errorf("failed to list partitions of table %s: query = [ %s ], error = %s", tableName, query, err) - utils.ErrExit("failed to find the partitions for table %s: %v", tableName, err) + utils.ErrExit("failed to find the partitions for table: %s: %v", tableName, err) } defer func() { closeErr := rows.Close() @@ -768,12 +768,12 @@ WHERE parent.relname='%s' AND nmsp_parent.nspname = '%s' `, tname, sname) var childSchema, childTable string err := rows.Scan(&childSchema, &childTable) if err != nil { - utils.ErrExit("Error in scanning for child partitions of table=%s: %v", tableName, err) + utils.ErrExit("Error in scanning for child partitions of table: %s: %v", tableName, err) } partitions = append(partitions, fmt.Sprintf(`%s.%s`, childSchema, childTable)) } if rows.Err() != nil { - utils.ErrExit("Error in scanning for child partitions of table=%s: %v", tableName, rows.Err()) + utils.ErrExit("Error in scanning for child partitions of table: %s: %v", tableName, rows.Err()) } return partitions } @@ -887,7 +887,7 @@ func (yb *YugabyteDB) GetNonPKTables() ([]string, error) { query := fmt.Sprintf(PG_QUERY_TO_CHECK_IF_TABLE_HAS_PK, querySchemaList) rows, err := yb.db.Query(query) if err != nil { - utils.ErrExit("error in querying(%q) source database for primary key: %v\n", query, err) + utils.ErrExit("error in querying source database for primary key: %q: %v\n", query, err) } defer func() { closeErr := rows.Close() diff --git a/yb-voyager/src/tgtdb/oracle.go b/yb-voyager/src/tgtdb/oracle.go index ce069d8407..6b69f6869a 100644 --- a/yb-voyager/src/tgtdb/oracle.go +++ b/yb-voyager/src/tgtdb/oracle.go @@ -160,7 +160,7 @@ func (tdb *TargetOracleDB) GetVersion() string { // query sample output: Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production err := tdb.QueryRow(query).Scan(&version) if err != nil { - utils.ErrExit("run query %q on source: %s", query, err) + utils.ErrExit("run query: %q on source: %s", query, err) } return version } @@ -184,7 +184,7 @@ func (tdb *TargetOracleDB) GetNonEmptyTables(tables []sqlname.NameTuple) []sqlna stmt := fmt.Sprintf("SELECT COUNT(*) FROM %s", table.ForUserQuery()) err := tdb.QueryRow(stmt).Scan(&rowCount) if err != nil { - utils.ErrExit("run query %q on target: %s", stmt, err) + utils.ErrExit("run query: %q on target: %s", stmt, err) } if rowCount > 0 { result = append(result, table) @@ -439,7 +439,7 @@ func (tdb *TargetOracleDB) setTargetSchema(conn *sql.Conn) { setSchemaQuery := fmt.Sprintf("ALTER SESSION SET CURRENT_SCHEMA = %s", tdb.tconf.Schema) _, err := conn.ExecContext(context.Background(), setSchemaQuery) if err != nil { - utils.ErrExit("run query %q on target %q to set schema: %s", setSchemaQuery, tdb.tconf.Host, err) + utils.ErrExit("run query: %q on target %q to set schema: %s", setSchemaQuery, tdb.tconf.Host, err) } } @@ -697,7 +697,7 @@ func (tdb *TargetOracleDB) isTableExists(nt sqlname.NameTuple) bool { func (tdb *TargetOracleDB) isQueryResultNonEmpty(query string) bool { rows, err := tdb.Query(query) if err != nil { - utils.ErrExit("error checking if query %s is empty: %v", query, err) + utils.ErrExit("error checking if query is empty: %q: %v", query, err) } defer rows.Close() diff --git a/yb-voyager/src/tgtdb/postgres.go b/yb-voyager/src/tgtdb/postgres.go index cb1dfcd217..53f3a78fee 100644 --- a/yb-voyager/src/tgtdb/postgres.go +++ b/yb-voyager/src/tgtdb/postgres.go @@ -336,7 +336,7 @@ func (pg *TargetPostgreSQL) GetNonEmptyTables(tables []sqlname.NameTuple) []sqln continue } if err != nil { - utils.ErrExit("failed to check whether table %q empty: %s", table, err) + utils.ErrExit("failed to check whether table is empty: %q: %s", table, err) } result = append(result, table) } @@ -656,7 +656,7 @@ func (pg *TargetPostgreSQL) setTargetSchema(conn *pgx.Conn) { setSchemaQuery := fmt.Sprintf("SET SEARCH_PATH TO %s", pg.tconf.Schema) _, err := conn.Exec(context.Background(), setSchemaQuery) if err != nil { - utils.ErrExit("run query %q on target %q: %s", setSchemaQuery, pg.tconf.Host, err) + utils.ErrExit("run query: %q on target %q: %s", setSchemaQuery, pg.tconf.Host, err) } } @@ -777,7 +777,7 @@ func (pg *TargetPostgreSQL) isTableExists(tableNameTup sqlname.NameTuple) bool { func (pg *TargetPostgreSQL) isQueryResultNonEmpty(query string) bool { rows, err := pg.Query(query) if err != nil { - utils.ErrExit("error checking if query %s is empty: %v", query, err) + utils.ErrExit("error checking if query is empty: %q: %v", query, err) } defer rows.Close() diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index cacf64120c..fc3a392ae9 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -404,7 +404,7 @@ func (yb *TargetYugabyteDB) GetNonEmptyTables(tables []sqlname.NameTuple) []sqln continue } if err != nil { - utils.ErrExit("failed to check whether table %q empty: %s", table, err) + utils.ErrExit("failed to check whether table is empty: %q: %s", table, err) } result = append(result, table) } @@ -1048,7 +1048,7 @@ func getYBSessionInitScript(tconf *TargetConf) []string { func checkSessionVariableSupport(tconf *TargetConf, sqlStmt string) bool { conn, err := pgx.Connect(context.Background(), tconf.GetConnectionUri()) if err != nil { - utils.ErrExit("error while creating connection for checking session parameter(%q) support: %v", sqlStmt, err) + utils.ErrExit("error while creating connection for checking session parameter support: %q: %v", sqlStmt, err) } defer conn.Close(context.Background()) @@ -1062,7 +1062,7 @@ func checkSessionVariableSupport(tconf *TargetConf, sqlStmt string) bool { } return true } - utils.ErrExit("error while executing sqlStatement=%q: %v", sqlStmt, err) + utils.ErrExit("error while executing sqlStatement: %q: %v", sqlStmt, err) } else { log.Warnf("Warning: %q is not supported: %v", sqlStmt, err) } @@ -1075,7 +1075,7 @@ func (yb *TargetYugabyteDB) setTargetSchema(conn *pgx.Conn) { setSchemaQuery := fmt.Sprintf("SET SCHEMA '%s'", yb.tconf.Schema) _, err := conn.Exec(context.Background(), setSchemaQuery) if err != nil { - utils.ErrExit("run query %q on target %q: %s", setSchemaQuery, yb.tconf.Host, err) + utils.ErrExit("run query: %q on target %q: %s", setSchemaQuery, yb.tconf.Host, err) } // append oracle schema in the search_path for orafce @@ -1214,7 +1214,7 @@ func (yb *TargetYugabyteDB) isTableExists(tableNameTup sqlname.NameTuple) bool { func (yb *TargetYugabyteDB) isQueryResultNonEmpty(query string) bool { rows, err := yb.Query(query) if err != nil { - utils.ErrExit("error checking if query %s is empty: %v", query, err) + utils.ErrExit("error checking if query is empty: [%s]: %v", query, err) } defer rows.Close() diff --git a/yb-voyager/src/utils/az/azutils.go b/yb-voyager/src/utils/az/azutils.go index b95576c709..dcbae81c55 100644 --- a/yb-voyager/src/utils/az/azutils.go +++ b/yb-voyager/src/utils/az/azutils.go @@ -38,7 +38,7 @@ func createClientIfNotExists(dataDir string) { var err error url, err := url.Parse(dataDir) if err != nil { - utils.ErrExit("parse azure blob url for dataDir %s: %w", dataDir, err) + utils.ErrExit("parse azure blob url: for dataDir %s: %w", dataDir, err) } serviceUrl := "https://" + url.Host // cred represents the default Oauth token used to authenticate the account in the url. diff --git a/yb-voyager/src/utils/s3/s3utils.go b/yb-voyager/src/utils/s3/s3utils.go index 5c335566b1..36cd6c7be6 100644 --- a/yb-voyager/src/utils/s3/s3utils.go +++ b/yb-voyager/src/utils/s3/s3utils.go @@ -95,7 +95,7 @@ func ListAllObjects(dataDir string) ([]string, error) { i++ page, err := p.NextPage(context.TODO()) if err != nil { - utils.ErrExit("failed to get page %v, %w", i, err) + utils.ErrExit("failed to get page %v: %w", i, err) } // Log the objects found for _, obj := range page.Contents { From 824dbf7270aa865d2f78cd843c4067c9e0ac25e0 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Thu, 9 Jan 2025 16:38:52 +0530 Subject: [PATCH 099/105] Reporting Collation - deterministic attribute and Merge statement (#2154) Reporting deterministic option (either true or false) as it is completely not supported in COLLATION object from PG. https://yugabyte.atlassian.net/browse/DB-14236 Reporting MERGE statement only as its >=PG15 feature. https://yugabyte.atlassian.net/browse/DB-14222 --- .../schema/collations/collation.sql | 8 ++- .../tests/analyze-schema/expected_issues.json | 20 +++++++ migtests/tests/analyze-schema/summary.json | 6 ++ .../expectedAssessmentReport.json | 50 +++++++++++++++- .../pg_assessment_report_uqc.sql | 22 +++++++ .../unsupported_query_constructs.sql | 10 ++++ .../expectedAssessmentReport.json | 16 +++++ .../pg_assessment_report.sql | 3 + yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/src/query/queryissue/constants.go | 5 ++ yb-voyager/src/query/queryissue/detectors.go | 32 +++++++++- .../src/query/queryissue/detectors_ddl.go | 23 +++++++ .../src/query/queryissue/detectors_test.go | 29 +++++++++ yb-voyager/src/query/queryissue/issues_ddl.go | 13 ++++ .../src/query/queryissue/issues_ddl_test.go | 15 +++++ yb-voyager/src/query/queryissue/issues_dml.go | 15 +++++ .../src/query/queryissue/issues_dml_test.go | 40 +++++++++++++ .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 26 ++++++-- .../src/query/queryparser/ddl_processor.go | 60 ++++++++++++++++--- .../src/query/queryparser/helpers_protomsg.go | 2 +- .../src/query/queryparser/helpers_struct.go | 51 ++++++++-------- .../src/query/queryparser/traversal_proto.go | 11 ++-- 23 files changed, 409 insertions(+), 50 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/collations/collation.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/collations/collation.sql index ed16fb0aa4..3b2898b004 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/collations/collation.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/collations/collation.sql @@ -1,3 +1,9 @@ --dropping multiple object -DROP COLLATION IF EXISTS coll1,coll2,coll3; \ No newline at end of file +DROP COLLATION IF EXISTS coll1,coll2,coll3; + +CREATE COLLATION special1 (provider = icu, locale = 'en@colCaseFirst=upper;colReorder=grek-latn', deterministic = true); + +CREATE COLLATION ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-true', deterministic = false); + + CREATE COLLATION schema2.upperfirst (provider = icu, locale = 'en-u-kf-upper'); \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 8baa84606e..f702baa5da 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -110,6 +110,26 @@ "GH": "", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "COLLATION", + "ObjectName": "special1", + "Reason": "Deterministic attribute in collation", + "SqlStatement": "CREATE COLLATION special1 (provider = icu, locale = 'en@colCaseFirst=upper;colReorder=grek-latn', deterministic = true);", + "Suggestion": "This feature is not supported in YugabyteDB yet", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "COLLATION", + "ObjectName": "ignore_accents", + "Reason": "Deterministic attribute in collation", + "SqlStatement": "CREATE COLLATION ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-true', deterministic = false);", + "Suggestion": "This feature is not supported in YugabyteDB yet", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 2052fbd69e..2a24c33dd6 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -11,6 +11,12 @@ "InvalidCount": 1, "ObjectNames": "hollywood" }, + { + "ObjectType": "COLLATION", + "TotalCount": 3, + "InvalidCount": 3, + "ObjectNames": "special1, ignore_accents, schema2.upperfirst" + }, { "ObjectType": "EXTENSION", "TotalCount": 7, diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index f9f859b2f0..790e5f0cd8 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -25,9 +25,16 @@ }, { "ObjectType": "TABLE", - "TotalCount": 5, + "TotalCount": 7, "InvalidCount": 2, - "ObjectNames": "analytics.metrics, sales.orders, sales.test_json_chk, sales.events, sales.json_data" + "ObjectNames": "analytics.metrics, sales.orders, sales.test_json_chk, sales.events, sales.json_data, sales.customer_account, sales.recent_transactions" + }, + { + "ObjectType": "SEQUENCE", + "TotalCount": 1, + "InvalidCount":0, + "ObjectNames": "sales.recent_transactions_transaction_id_seq" + }, { "ObjectType": "VIEW", @@ -48,11 +55,13 @@ "ColocatedTables": [ "sales.orders", "analytics.metrics", + "sales.customer_account", + "sales.recent_transactions", "sales.events", "sales.json_data", "sales.test_json_chk" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 5 objects (5 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 7 objects (7 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": null, "NumNodes": 3, "VCPUsPerInstance": 4, @@ -115,6 +124,34 @@ ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ + { + "SchemaName": "sales", + "ObjectName": "customer_account", + "RowCount": 4, + "ColumnCount": 2, + "Reads": 7, + "Writes": 6, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, + { + "SchemaName": "sales", + "ObjectName": "recent_transactions", + "RowCount": 3, + "ColumnCount": 3, + "Reads": 3, + "Writes": 3, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, { "SchemaName": "sales", "ObjectName": "test_json_chk", @@ -214,6 +251,13 @@ "MinimumVersionsFixedIn": null }, { + "ConstructTypeName": "Merge Statement", + "Query": "MERGE INTO sales.customer_account ca\nUSING sales.recent_transactions t \nON t.customer_id = ca.customer_id\nWHEN MATCHED THEN\n UPDATE SET balance = balance + transaction_value\nWHEN NOT MATCHED THEN\n INSERT (customer_id, balance)\n VALUES (t.customer_id, t.transaction_value)", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Jsonb Subscripting", "Query": "SELECT \n data,\n data[$1] AS name, \n (data[$2]) as active\nFROM sales.test_json_chk", "DocsLink": "", diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index 987f4f05b4..bb31768d8b 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -47,6 +47,28 @@ create view sales.employ_depart_view AS SELECT any_value(name) AS any_employee FROM employees; +CREATE TABLE sales.customer_account ( + customer_id INT PRIMARY KEY, + balance NUMERIC(10, 2) NOT NULL +); + +INSERT INTO sales.customer_account (customer_id, balance) +VALUES + (1, 100.00), + (2, 200.00), + (3, 300.00); + +CREATE TABLE sales.recent_transactions ( + transaction_id SERIAL PRIMARY KEY, + customer_id INT NOT NULL, + transaction_value NUMERIC(10, 2) NOT NULL +); + +INSERT INTO sales.recent_transactions (customer_id, transaction_value) +VALUES + (1, 50.00), + (3, -25.00), + (4, 150.00); CREATE TABLE sales.test_json_chk ( id int, name text, diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index d2f4f089e3..9133844f14 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -27,6 +27,16 @@ SELECT any_value(name) AS any_employee FROM employees; +MERGE INTO sales.customer_account ca +USING sales.recent_transactions t +ON t.customer_id = ca.customer_id +WHEN MATCHED THEN + UPDATE SET balance = balance + transaction_value +WHEN NOT MATCHED THEN + INSERT (customer_id, balance) + VALUES (t.customer_id, t.transaction_value); + +select * from sales.customer_account ; SELECT (sales.get_user_info(2))['name'] AS user_info; SELECT (jsonb_build_object('name', 'PostgreSQL', 'version', 17, 'open_source', TRUE) || '{"key": "value2"}')['name'] AS json_obj; diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 9ded5a5edf..58c8596964 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -100,6 +100,12 @@ "TotalCount": 4, "InvalidCount": 2, "ObjectNames": "policy_test_fine ON public.test_exclude_basic, policy_test_fine_2 ON public.employees2, policy_test_report ON public.test_xml_type, policy_test_report ON schema2.test_xml_type" + }, + { + "ObjectType": "COLLATION", + "TotalCount": 2, + "InvalidCount": 1, + "ObjectNames": "schema2.ignore_accents, public.\"numeric\"" } ] }, @@ -583,6 +589,16 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", "MinimumVersionsFixedIn": null }, + { + "FeatureName": "Deterministic attribute in collation", + "Objects": [ + { + "ObjectName": "schema2.ignore_accents", + "SqlStatement": "CREATE COLLATION schema2.ignore_accents (provider = icu, deterministic = false, locale = 'und-u-kc-ks-level1');" + } + ], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Primary / Unique key constraints on complex datatypes", "Objects": [ diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index e8882d3a91..c0ffcb0d5d 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -430,6 +430,9 @@ WITH (security_invoker = true) AS SELECT employee_id, first_name FROM public.employees; +CREATE COLLATION schema2.ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-true', deterministic = false); + + CREATE COLLATION public.numeric (provider = icu, locale = 'en@colNumeric=yes'); -- Testing tables with unique nulls not distinct constraints -- Control case diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 357c900ff4..e16e0efc62 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1067,6 +1067,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.DETERMINISTIC_OPTION_WITH_COLLATION_NAME, "", queryissue.DETERMINISTIC_OPTION_WITH_COLLATION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.UNIQUE_NULLS_NOT_DISTINCT_NAME, "", queryissue.UNIQUE_NULLS_NOT_DISTINCT, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 2db0cab2e2..80cce2e1da 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -79,6 +79,11 @@ const ( COPY_FROM_WHERE = "COPY FROM ... WHERE" COPY_ON_ERROR = "COPY ... ON_ERROR" + DETERMINISTIC_OPTION_WITH_COLLATION = "DETERMINISTIC_OPTION_WITH_COLLATION" + DETERMINISTIC_OPTION_WITH_COLLATION_NAME = "Deterministic attribute in collation" + + MERGE_STATEMENT = "MERGE_STATEMENT" + MERGE_STATEMENT_NAME = "Merge Statement" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE = "FOREIGN_KEY_REFERENCED_PARTITIONED_TABLE" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME = "Foreign key constraint references partitioned table" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 1ba0a53362..c62f473edb 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -317,7 +317,7 @@ func NewCopyCommandUnsupportedConstructsDetector(query string) *CopyCommandUnsup // Detect if COPY command uses unsupported syntax i.e. COPY FROM ... WHERE and COPY... ON_ERROR func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) error { // Check if the message is a COPY statement - if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPYSTSMT_NODE { + if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPY_STMT_NODE { return nil // Not a COPY statement, nothing to detect } @@ -453,6 +453,33 @@ func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { return issues } +type MergeStatementDetector struct { + query string + isMergeStatementDetected bool +} + +func NewMergeStatementDetector(query string) *MergeStatementDetector { + return &MergeStatementDetector{ + query: query, + } +} + +func (m *MergeStatementDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_MERGE_STMT_NODE { + m.isMergeStatementDetected = true + } + return nil + +} + +func (m *MergeStatementDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if m.isMergeStatementDetected { + issues = append(issues, NewMergeStatementIssue(DML_QUERY_OBJECT_TYPE, "", m.query)) + } + return issues +} + type UniqueNullsNotDistinctDetector struct { query string detected bool @@ -466,7 +493,7 @@ func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDete // Detect checks if a unique constraint is defined which has nulls not distinct func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { - if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEXSTMT_NODE { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEX_STMT_NODE { indexStmt, err := queryparser.ProtoAsIndexStmt(msg) if err != nil { return err @@ -507,7 +534,6 @@ func NewJsonPredicateExprDetector(query string) *JsonPredicateExprDetector { query: query, } } - func (j *JsonPredicateExprDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_IS_PREDICATE_NODE { /* diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 7a5401976b..a5259aaeb6 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -611,6 +611,27 @@ func (v *MViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss return nil, nil } +//===============COLLATION ISSUE DETECTOR ==================== + +type CollationIssueDetector struct{} + +func (c *CollationIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { + collation, ok := obj.(*queryparser.Collation) + if !ok { + return nil, fmt.Errorf("invalid object type: expected Collation") + } + issues := make([]QueryIssue, 0) + if slices.Contains(collation.Options, "deterministic") { + // deterministic attribute is itself not supported in YB either true or false so checking only whether option is present or not + issues = append(issues, NewDeterministicOptionCollationIssue( + collation.GetObjectType(), + collation.GetObjectName(), + "", + )) + } + return issues, nil +} + //=============NO-OP ISSUE DETECTOR =========================== // Need to handle all the cases for which we don't have any issues detector @@ -646,6 +667,8 @@ func (p *ParserIssueDetector) GetDDLDetector(obj queryparser.DDLObject) (DDLIssu return &ViewIssueDetector{}, nil case *queryparser.MView: return &MViewIssueDetector{}, nil + case *queryparser.Collation: + return &CollationIssueDetector{}, nil default: return &NoOpIssueDetector{}, nil } diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 08aaf1b4af..35db47c70f 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -734,6 +734,35 @@ WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);` } +func TestMergeStatementDetector(t *testing.T) { + sqls := []string{ + `MERGE INTO customer_account ca +USING recent_transactions t +ON t.customer_id = ca.customer_id +WHEN MATCHED THEN + UPDATE SET balance = balance + transaction_value +WHEN NOT MATCHED THEN + INSERT (customer_id, balance) + VALUES (t.customer_id, t.transaction_value);`, + + `MERGE INTO wines w +USING wine_stock_changes s +ON s.winename = w.winename +WHEN NOT MATCHED AND s.stock_delta > 0 THEN + INSERT VALUES(s.winename, s.stock_delta) +WHEN MATCHED AND w.stock + s.stock_delta > 0 THEN + UPDATE SET stock = w.stock + s.stock_delta +WHEN MATCHED THEN + DELETE +RETURNING merge_action(), w.*;`, + } + + for _, sql := range sqls { + issues := getDetectorIssues(t, NewMergeStatementDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, MERGE_STATEMENT, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + } +} func TestIsJsonPredicate(t *testing.T) { sql := `SELECT js, js IS JSON "json?" FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);` diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index c6eb5f3955..468ef2bc5c 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -500,6 +500,19 @@ func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatem return newQueryIssue(securityInvokerViewIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) } +var deterministicOptionCollationIssue = issue.Issue{ + Type: DETERMINISTIC_OPTION_WITH_COLLATION, + Name: DETERMINISTIC_OPTION_WITH_COLLATION_NAME, + Impact: constants.IMPACT_LEVEL_1, + Suggestion: "This feature is not supported in YugabyteDB yet", + GH: "", // TODO + DocsLink: "", // TODO +} + +func NewDeterministicOptionCollationIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(deterministicOptionCollationIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + var foreignKeyReferencesPartitionedTableIssue = issue.Issue{ Type: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, Name: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 53498dffa8..01f90aa0fd 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -274,6 +274,18 @@ func testSecurityInvokerView(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "unrecognized parameter", securityInvokerViewIssue) } +func testDeterministicCollationIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE COLLATION case_insensitive (provider = icu, locale = 'und-u-ks-level2', deterministic = false);`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `collation attribute "deterministic" not recognized`, securityInvokerViewIssue) +} + func testForeignKeyReferencesPartitionedTableIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -363,6 +375,9 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "security invoker view", ybVersion), testSecurityInvokerView) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "deterministic attribute in collation", ybVersion), testDeterministicCollationIssue) + assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "foreign key referenced partitioned table", ybVersion), testForeignKeyReferencesPartitionedTableIssue) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 393841a79e..322863f4a3 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -220,3 +220,18 @@ var fetchWithTiesIssue = issue.Issue{ func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { return newQueryIssue(fetchWithTiesIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } + +var mergeStatementIssue = issue.Issue{ + Type: MERGE_STATEMENT, + Name: "Merge Statement", + Impact: constants.IMPACT_LEVEL_2, + Description: "This statement is not supported in YugabyteDB yet", + Suggestion: "Use PL/pgSQL to write the logic to get this functionality", + GH: "", + DocsLink: "", //TODO +} + +func NewMergeStatementIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + //MERGE STATEMENT is PG15 feature but MERGE .... RETURNING clause is PG17 feature so need to report it separately later. + return newQueryIssue(mergeStatementIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index d9a0b8209b..c18352fc75 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -181,6 +181,43 @@ WHERE JSON_EXISTS(details, '$.author');`, assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "COLUMNS"`, jsonConstructorFunctionsIssue) } +func testMergeStmtIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + sqls := []string{` + MERGE INTO customer_account ca +USING recent_transactions t +ON t.customer_id = ca.customer_id +WHEN MATCHED THEN + UPDATE SET balance = balance + transaction_value +WHEN NOT MATCHED THEN + INSERT (customer_id, balance) + VALUES (t.customer_id, t.transaction_value); +`, + ` + MERGE INTO wines w +USING wine_stock_changes s +ON s.winename = w.winename +WHEN NOT MATCHED AND s.stock_delta > 0 THEN + INSERT VALUES(s.winename, s.stock_delta) +WHEN MATCHED AND w.stock + s.stock_delta > 0 THEN + UPDATE SET stock = w.stock + s.stock_delta +WHEN MATCHED THEN + DELETE +RETURNING merge_action(), w.*; + `, // MERGE ... RETURNING statement >PG15 feature + } + + for _, sql := range sqls { + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "MERGE"`, mergeStatementIssue) + } + +} + func testAggFunctions(t *testing.T) { sqls := []string{ `CREATE TABLE any_value_ex ( @@ -275,6 +312,9 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "json query functions", ybVersion), testJsonQueryFunctions) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "merge statement", ybVersion), testMergeStmtIssue) + assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "json subscripting", ybVersion), testJsonbSubscriptingIssue) assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "aggregate functions", ybVersion), testAggFunctions) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index e452f34a38..5fbc47ca8e 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -393,6 +393,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), + NewMergeStatementDetector(query), NewJsonbSubscriptingDetector(query, p.jsonbColumns, p.getJsonbReturnTypeFunctions()), NewUniqueNullsNotDistinctDetector(query), NewJsonPredicateExprDetector(query), diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 97e364f922..744abc25f8 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -163,14 +163,18 @@ CHECK (xpath_exists('/invoice/customer', data));` WITH (security_invoker = true) AS SELECT employee_id, first_name FROM public.employees;` - stmt21 = `CREATE TABLE public.products ( + stmt21 = `CREATE COLLATION case_insensitive (provider = icu, locale = 'und-u-ks-level2', deterministic = false);` + stmt22 = `CREATE COLLATION new_schema.ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-true', deterministic = false);` + stmt23 = `CREATE COLLATION upperfirst (provider = icu, locale = 'en-u-kf-upper', deterministic = true);` + stmt24 = `CREATE COLLATION special (provider = icu, locale = 'en-u-kf-upper-kr-grek-latn');` + stmt25 = `CREATE TABLE public.products ( id INTEGER PRIMARY KEY, product_name VARCHAR(100), serial_number TEXT, UNIQUE NULLS NOT DISTINCT (product_name, serial_number) );` - stmt22 = `ALTER TABLE public.products ADD CONSTRAINT unique_product_name UNIQUE NULLS NOT DISTINCT (product_name);` - stmt23 = `CREATE UNIQUE INDEX unique_email_idx ON users (email) NULLS NOT DISTINCT;` + stmt26 = `ALTER TABLE public.products ADD CONSTRAINT unique_product_name UNIQUE NULLS NOT DISTINCT (product_name);` + stmt27 = `CREATE UNIQUE INDEX unique_email_idx ON users (email) NULLS NOT DISTINCT;` ) func modifiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { @@ -295,13 +299,23 @@ func TestDDLIssues(t *testing.T) { NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), }, stmt21: []QueryIssue{ - NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt21), + NewDeterministicOptionCollationIssue("COLLATION", "case_insensitive", stmt21), }, stmt22: []QueryIssue{ - NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt22), + NewDeterministicOptionCollationIssue("COLLATION", "new_schema.ignore_accents", stmt22), }, stmt23: []QueryIssue{ - NewUniqueNullsNotDistinctIssue("INDEX", "unique_email_idx ON users", stmt23), + NewDeterministicOptionCollationIssue("COLLATION", "upperfirst", stmt23), + }, + stmt24: []QueryIssue{}, + stmt25: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt25), + }, + stmt26: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt26), + }, + stmt27: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("INDEX", "unique_email_idx ON users", stmt27), }, } for _, stmt := range requiredDDLs { diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 412dc4c8c8..895d6f518f 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -157,7 +157,7 @@ func (tableProcessor *TableProcessor) parseTableElts(tableElts []*pg_query.Node, colName := element.GetColumnDef().GetColname() typeNames := element.GetColumnDef().GetTypeName().GetNames() - typeName, typeSchemaName := getTypeNameAndSchema(typeNames) + typeSchemaName, typeName := getSchemaAndObjectName(typeNames) /* e.g. CREATE TABLE test_xml_type(id int, data xml); relation:{relname:"test_xml_type" inh:true relpersistence:"p" location:15} table_elts:{column_def:{colname:"id" @@ -411,7 +411,7 @@ func (ftProcessor *ForeignTableProcessor) Process(parseTree *pg_query.ParseResul colName := element.GetColumnDef().GetColname() typeNames := element.GetColumnDef().GetTypeName().GetNames() - typeName, typeSchemaName := getTypeNameAndSchema(typeNames) + typeSchemaName, typeName := getSchemaAndObjectName(typeNames) table.Columns = append(table.Columns, TableColumn{ ColumnName: colName, TypeName: typeName, @@ -496,7 +496,7 @@ func (indexProcessor *IndexProcessor) parseIndexParams(params []*pg_query.Node) if ip.IsExpression { //For the expression index case to report in case casting to unsupported types #3 typeNames := i.GetIndexElem().GetExpr().GetTypeCast().GetTypeName().GetNames() - ip.ExprCastTypeName, ip.ExprCastTypeSchema = getTypeNameAndSchema(typeNames) + ip.ExprCastTypeSchema, ip.ExprCastTypeName = getSchemaAndObjectName(typeNames) ip.IsExprCastArrayType = isArrayType(i.GetIndexElem().GetExpr().GetTypeCast().GetTypeName()) } indexParams = append(indexParams, ip) @@ -770,7 +770,7 @@ func (triggerProcessor *TriggerProcessor) Process(parseTree *pg_query.ParseResul Events: triggerNode.CreateTrigStmt.Events, ForEachRow: triggerNode.CreateTrigStmt.Row, } - _, trigger.FuncName = getFunctionObjectName(triggerNode.CreateTrigStmt.Funcname) + _, trigger.FuncName = getSchemaAndObjectName(triggerNode.CreateTrigStmt.Funcname) return trigger, nil } @@ -848,7 +848,7 @@ func (typeProcessor *TypeProcessor) Process(parseTree *pg_query.ParseResult) (DD return createType, nil case isEnum: typeNames := enumNode.CreateEnumStmt.GetTypeName() - typeName, typeSchemaName := getTypeNameAndSchema(typeNames) + typeSchemaName, typeName := getSchemaAndObjectName(typeNames) createType := &CreateType{ TypeName: typeName, SchemaName: typeSchemaName, @@ -962,6 +962,45 @@ func (mv *MView) GetSchemaName() string { return mv.SchemaName } func (mv *MView) GetObjectType() string { return MVIEW_OBJECT_TYPE } +//=============================COLLATION PROCESSOR ============== + +type CollationProcessor struct{} + +func NewCollationProcessor() *CollationProcessor { + return &CollationProcessor{} +} + +func (cp *CollationProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, error) { + defineStmt, ok := getDefineStmtNode(parseTree) + if !ok { + return nil, fmt.Errorf("not a CREATE COLLATION statement") + } + schema, colName := getSchemaAndObjectName(defineStmt.Defnames) + defNames, err := TraverseAndExtractDefNamesFromDefElem(defineStmt.ProtoReflect()) + if err != nil { + return nil, fmt.Errorf("error getting the defElems in collation: %v", err) + } + collation := Collation{ + SchemaName: schema, + CollationName: colName, + Options: defNames, + } + return &collation, nil +} + +type Collation struct { + SchemaName string + CollationName string + Options []string +} + +func (c *Collation) GetObjectName() string { + return lo.Ternary(c.SchemaName != "", fmt.Sprintf("%s.%s", c.SchemaName, c.CollationName), c.CollationName) +} +func (c *Collation) GetSchemaName() string { return c.SchemaName } + +func (c *Collation) GetObjectType() string { return COLLATION_OBJECT_TYPE } + // ============================Function Processor ================= type FunctionProcessor struct{} @@ -977,7 +1016,7 @@ func (mv *FunctionProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject } funcNameList := funcNode.CreateFunctionStmt.GetFuncname() - funcSchemaName, funcName := getFunctionObjectName(funcNameList) + funcSchemaName, funcName := getSchemaAndObjectName(funcNameList) function := Function{ SchemaName: funcSchemaName, FuncName: funcName, @@ -1044,9 +1083,13 @@ func GetDDLProcessor(parseTree *pg_query.ParseResult) (DDLProcessor, error) { case PG_QUERY_CREATE_TABLE_AS_STMT: if IsMviewObject(parseTree) { return NewMViewProcessor(), nil - } else { - return NewNoOpProcessor(), nil } + return NewNoOpProcessor(), nil + case PG_QUERY_DEFINE_STMT_NODE: + if IsCollationObject(parseTree) { + return NewCollationProcessor(), nil + } + return NewNoOpProcessor(), nil case PG_QUERY_CREATE_FUNCTION_STMT: return NewFunctionProcessor(), nil default: @@ -1065,6 +1108,7 @@ const ( INDEX_OBJECT_TYPE = "INDEX" POLICY_OBJECT_TYPE = "POLICY" TRIGGER_OBJECT_TYPE = "TRIGGER" + COLLATION_OBJECT_TYPE = "COLLATION" ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 7889fb9529..174270d72b 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -427,7 +427,7 @@ func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { } indexStmtNode, ok := protoMsg.(*pg_query.IndexStmt) if !ok { - return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEXSTMT_NODE) + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEX_STMT_NODE) } return indexStmtNode, nil } diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 1efbe99dac..46d02b5edb 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -45,6 +45,22 @@ func IsMviewObject(parseTree *pg_query.ParseResult) bool { return isCreateAsStmt && createAsNode.CreateTableAsStmt.Objtype == pg_query.ObjectType_OBJECT_MATVIEW } +func getDefineStmtNode(parseTree *pg_query.ParseResult) (*pg_query.DefineStmt, bool) { + node, ok := parseTree.Stmts[0].Stmt.Node.(*pg_query.Node_DefineStmt) + return node.DefineStmt, ok +} + +func IsCollationObject(parseTree *pg_query.ParseResult) bool { + collation, ok := getDefineStmtNode(parseTree) + /* + stmts:{stmt:{define_stmt:{kind:OBJECT_COLLATION defnames:{string:{sval:"ignore_accents"}} definition:{def_elem:{defname:"provider" + arg:{type_name:{names:{string:{sval:"icu"}} typemod:-1 location:48}} defaction:DEFELEM_UNSPEC location:37}} definition:{def_elem:{defname:"locale" + arg:{string:{sval:"und-u-ks-level1-kc-true"}} defaction:DEFELEM_UNSPEC location:55}} definition:{def_elem:{defname:"deterministic" + arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:91}}}} stmt_len:113} + */ + return ok && collation.Kind == pg_query.ObjectType_OBJECT_COLLATION +} + func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string) { createFuncNode, isCreateFunc := getCreateFuncStmtNode(parseTree) viewNode, isViewStmt := getCreateViewNode(parseTree) @@ -69,7 +85,7 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string objectType = "PROCEDURE" } funcNameList := stmt.GetFuncname() - funcSchemaName, funcName := getFunctionObjectName(funcNameList) + funcSchemaName, funcName := getSchemaAndObjectName(funcNameList) return objectType, utils.BuildObjectName(funcSchemaName, funcName) case isViewStmt: viewName := viewNode.ViewStmt.View @@ -104,29 +120,16 @@ func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string { return utils.BuildObjectName(schema, name) } -func getFunctionObjectName(funcNameList []*pg_query.Node) (string, string) { - funcName := "" - funcSchemaName := "" - if len(funcNameList) > 0 { - funcName = funcNameList[len(funcNameList)-1].GetString_().Sval // func name can be qualified / unqualifed or native / non-native proper func name will always be available at last index +func getSchemaAndObjectName(nameList []*pg_query.Node) (string, string) { + objName := "" + schemaName := "" + if len(nameList) > 0 { + objName = nameList[len(nameList)-1].GetString_().Sval // obj name can be qualified / unqualifed or native / non-native proper func name will always be available at last index } - if len(funcNameList) >= 2 { // Names list will have all the parts of qualified func name - funcSchemaName = funcNameList[len(funcNameList)-2].GetString_().Sval // // func name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index + if len(nameList) >= 2 { // Names list will have all the parts of qualified func name + schemaName = nameList[len(nameList)-2].GetString_().Sval // // obj name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index } - return funcSchemaName, funcName -} - -func getTypeNameAndSchema(typeNames []*pg_query.Node) (string, string) { - typeName := "" - typeSchemaName := "" - if len(typeNames) > 0 { - typeName = typeNames[len(typeNames)-1].GetString_().Sval // type name can be qualified / unqualifed or native / non-native proper type name will always be available at last index - } - if len(typeNames) >= 2 { // Names list will have all the parts of qualified type name - typeSchemaName = typeNames[len(typeNames)-2].GetString_().Sval // // type name can be qualified / unqualifed or native / non-native proper schema name will always be available at last 2nd index - } - - return typeName, typeSchemaName + return schemaName, objName } func getCreateTableAsStmtNode(parseTree *pg_query.ParseResult) (*pg_query.Node_CreateTableAsStmt, bool) { @@ -298,7 +301,7 @@ func DoesNodeHandleJsonbData(node *pg_query.Node, jsonbColumns []string, jsonbFu type_name:{names:{string:{sval:"jsonb"}} typemod:-1 location:306} location:304}} */ typeCast := node.GetTypeCast() - typeName, _ := getTypeNameAndSchema(typeCast.GetTypeName().GetNames()) + _, typeName := getSchemaAndObjectName(typeCast.GetTypeName().GetNames()) if typeName == "jsonb" { return true } @@ -310,7 +313,7 @@ func DoesNodeHandleJsonbData(node *pg_query.Node, jsonbColumns []string, jsonbFu args:{a_const:{ival:{ival:14} location:227}} */ funcCall := node.GetFuncCall() - _, funcName := getFunctionObjectName(funcCall.Funcname) + _, funcName := getSchemaAndObjectName(funcCall.Funcname) if slices.Contains(jsonbFunctions, funcName) { return true } diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 9bf2b2716f..09b2658ac7 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -51,10 +51,13 @@ const ( PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" PG_QUERY_JSON_IS_PREDICATE_NODE = "pg_query.JsonIsPredicate" - PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" - PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" - PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" - PG_QUERY_INDEXSTMT_NODE = "pg_query.IndexStmt" + PG_QUERY_VIEW_STMT_NODE = "pg_query.ViewStmt" + PG_QUERY_COPY_STMT_NODE = "pg_query.CopyStmt" + + PG_QUERY_DEFINE_STMT_NODE = "pg_query.DefineStmt" + PG_QUERY_MERGE_STMT_NODE = "pg_query.MergeStmt" + PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" + PG_QUERY_INDEX_STMT_NODE = "pg_query.IndexStmt" ) // function type for processing nodes during traversal From 7802ecbc995236dc5e9a22b996f267431c7f2cb2 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 9 Jan 2025 17:29:42 +0530 Subject: [PATCH 100/105] Implement Migration Complexity Explanation providing the summary and rationale for reported complexity (#2166) * For now, Ignoring MigrationComplexityExplanation field in the tests to be verified --- migtests/scripts/functions.sh | 1 + yb-voyager/cmd/assessMigrationCommand.go | 15 ++ yb-voyager/cmd/common.go | 17 +- yb-voyager/cmd/common_test.go | 40 ++-- yb-voyager/cmd/migration_complexity.go | 181 ++++++++++++++++-- .../migration_assessment_report.template | 5 + yb-voyager/src/utils/utils.go | 13 ++ 7 files changed, 225 insertions(+), 47 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 3b80a1ea7c..f01d384a06 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -893,6 +893,7 @@ normalize_json() { .OptimalSelectConnectionsPerNode? = "IGNORED" | .OptimalInsertConnectionsPerNode? = "IGNORED" | .RowCount? = "IGNORED" | + .MigrationComplexityExplanation?= "IGNORED" | # Replace newline characters in SqlStatement with spaces .SqlStatement? |= ( if type == "string" then diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index e16e0efc62..586610676f 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -424,6 +424,7 @@ func assessMigration() (err error) { } log.Infof("number of assessment issues detected: %d\n", len(assessmentReport.Issues)) + utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() controlPlane.MigrationAssessmentCompleted(completedEvent) @@ -1603,6 +1604,14 @@ func postProcessingOfAssessmentReport() { func generateAssessmentReportJson(reportDir string) error { jsonReportFilePath := filepath.Join(reportDir, fmt.Sprintf("%s%s", ASSESSMENT_FILE_NAME, JSON_EXTENSION)) log.Infof("writing assessment report to file: %s", jsonReportFilePath) + + var err error + assessmentReport.MigrationComplexityExplanation, err = buildMigrationComplexityExplanation(source.DBType, assessmentReport, "") + if err != nil { + return fmt.Errorf("unable to build migration complexity explanation for json report: %w", err) + } + log.Info(assessmentReport.MigrationComplexityExplanation) + strReport, err := json.MarshalIndent(assessmentReport, "", "\t") if err != nil { return fmt.Errorf("failed to marshal the assessment report: %w", err) @@ -1621,6 +1630,12 @@ func generateAssessmentReportHtml(reportDir string) error { htmlReportFilePath := filepath.Join(reportDir, fmt.Sprintf("%s%s", ASSESSMENT_FILE_NAME, HTML_EXTENSION)) log.Infof("writing assessment report to file: %s", htmlReportFilePath) + var err error + assessmentReport.MigrationComplexityExplanation, err = buildMigrationComplexityExplanation(source.DBType, assessmentReport, "html") + if err != nil { + return fmt.Errorf("unable to build migration complexity explanation for html report: %w", err) + } + file, err := os.Create(htmlReportFilePath) if err != nil { return fmt.Errorf("failed to create file for %q: %w", filepath.Base(htmlReportFilePath), err) diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 7c9394d432..4c9f34138c 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1051,14 +1051,15 @@ func storeTableListInMSR(tableList []sqlname.NameTuple) error { // TODO: consider merging all unsupported field with single AssessmentReport struct member as AssessmentIssue type AssessmentReport struct { - VoyagerVersion string `json:"VoyagerVersion"` - TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` - MigrationComplexity string `json:"MigrationComplexity"` - SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` - Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` - Issues []AssessmentIssue `json:"-"` // disabled in reports till corresponding UI changes are done(json and html reports) - TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` - Notes []string `json:"Notes"` + VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` + MigrationComplexity string `json:"MigrationComplexity"` + MigrationComplexityExplanation string `json:"MigrationComplexityExplanation"` + SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` + Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` + Issues []AssessmentIssue `json:"-"` // disabled in reports till corresponding UI changes are done(json and html reports) + TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` + Notes []string `json:"Notes"` // fields going to be deprecated UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index 6dd45e6669..9fd9635f3a 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -129,21 +129,22 @@ func TestAssessmentReportStructs(t *testing.T) { name: "Validate AssessmentReport Struct Definition", actualType: reflect.TypeOf(AssessmentReport{}), expectedType: struct { - VoyagerVersion string `json:"VoyagerVersion"` - TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` - MigrationComplexity string `json:"MigrationComplexity"` - SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` - Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` - Issues []AssessmentIssue `json:"-"` - TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` - Notes []string `json:"Notes"` - UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` - UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` - UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` - UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` - UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` - UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` - MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` + VoyagerVersion string `json:"VoyagerVersion"` + TargetDBVersion *ybversion.YBVersion `json:"TargetDBVersion"` + MigrationComplexity string `json:"MigrationComplexity"` + MigrationComplexityExplanation string `json:"MigrationComplexityExplanation"` + SchemaSummary utils.SchemaSummary `json:"SchemaSummary"` + Sizing *migassessment.SizingAssessmentReport `json:"Sizing"` + Issues []AssessmentIssue `json:"-"` + TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"` + Notes []string `json:"Notes"` + UnsupportedDataTypes []utils.TableColumnsDataTypes `json:"UnsupportedDataTypes"` + UnsupportedDataTypesDesc string `json:"UnsupportedDataTypesDesc"` + UnsupportedFeatures []UnsupportedFeature `json:"UnsupportedFeatures"` + UnsupportedFeaturesDesc string `json:"UnsupportedFeaturesDesc"` + UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"` + UnsupportedPlPgSqlObjects []UnsupportedFeature `json:"UnsupportedPlPgSqlObjects"` + MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` }{}, }, } @@ -165,9 +166,10 @@ func TestAssessmentReportJson(t *testing.T) { } assessmentReport = AssessmentReport{ - VoyagerVersion: "v1.0.0", - TargetDBVersion: newYbVersion, - MigrationComplexity: "High", + VoyagerVersion: "v1.0.0", + TargetDBVersion: newYbVersion, + MigrationComplexity: "High", + MigrationComplexityExplanation: "", SchemaSummary: utils.SchemaSummary{ Description: "Test Schema Summary", DBName: "test_db", @@ -302,12 +304,12 @@ func TestAssessmentReportJson(t *testing.T) { if err != nil { t.Fatalf("Failed to write assessment report to JSON file: %v", err) } - // expected JSON expectedJSON := `{ "VoyagerVersion": "v1.0.0", "TargetDBVersion": "2024.1.1.1", "MigrationComplexity": "High", + "MigrationComplexityExplanation": "", "SchemaSummary": { "Description": "Test Schema Summary", "DbName": "test_db", diff --git a/yb-voyager/cmd/migration_complexity.go b/yb-voyager/cmd/migration_complexity.go index ae0b379759..69b71d1d3b 100644 --- a/yb-voyager/cmd/migration_complexity.go +++ b/yb-voyager/cmd/migration_complexity.go @@ -16,12 +16,14 @@ limitations under the License. package cmd import ( + "bytes" "encoding/csv" "fmt" "math" "os" "path/filepath" "strings" + "text/template" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -33,15 +35,16 @@ import ( const NOT_AVAILABLE = "NOT AVAILABLE" var ( - LEVEL_1_MEDIUM_THRESHOLD = 20 - LEVEL_1_HIGH_THRESHOLD = math.MaxInt32 - LEVEL_2_MEDIUM_THRESHOLD = 10 - LEVEL_2_HIGH_THRESHOLD = 100 - LEVEL_3_MEDIUM_THRESHOLD = 0 - LEVEL_3_HIGH_THRESHOLD = 4 + LEVEL_1_MEDIUM_THRESHOLD = 20 + LEVEL_1_HIGH_THRESHOLD = math.MaxInt32 + LEVEL_2_MEDIUM_THRESHOLD = 10 + LEVEL_2_HIGH_THRESHOLD = 100 + LEVEL_3_MEDIUM_THRESHOLD = 0 + LEVEL_3_HIGH_THRESHOLD = 4 + migrationComplexityRationale string ) -// Migration complexity calculation from the conversion issues +// Migration complexity calculation based on the detected assessment issues func calculateMigrationComplexity(sourceDBType string, schemaDirectory string, assessmentReport AssessmentReport) string { if sourceDBType != ORACLE && sourceDBType != POSTGRESQL { return NOT_AVAILABLE @@ -64,30 +67,37 @@ func calculateMigrationComplexity(sourceDBType string, schemaDirectory string, a } func calculateMigrationComplexityForPG(assessmentReport AssessmentReport) string { + if assessmentReport.MigrationComplexity != "" { + return assessmentReport.MigrationComplexity + } + counts := lo.CountValuesBy(assessmentReport.Issues, func(issue AssessmentIssue) string { return issue.Impact }) + l1IssueCount := counts[constants.IMPACT_LEVEL_1] + l2IssueCount := counts[constants.IMPACT_LEVEL_2] + l3IssueCount := counts[constants.IMPACT_LEVEL_3] - level1IssueCount := counts[constants.IMPACT_LEVEL_1] - level2IssueCount := counts[constants.IMPACT_LEVEL_2] - level3IssueCount := counts[constants.IMPACT_LEVEL_3] + log.Infof("issue counts: level-1=%d, level-2=%d, level-3=%d\n", l1IssueCount, l2IssueCount, l3IssueCount) - utils.PrintAndLog("issue counts: level-1=%d, level-2=%d, level-3=%d\n", level1IssueCount, level2IssueCount, level3IssueCount) // Determine complexity for each level - comp1 := getComplexityForLevel(constants.IMPACT_LEVEL_1, level1IssueCount) - comp2 := getComplexityForLevel(constants.IMPACT_LEVEL_2, level2IssueCount) - comp3 := getComplexityForLevel(constants.IMPACT_LEVEL_3, level3IssueCount) + comp1 := getComplexityForLevel(constants.IMPACT_LEVEL_1, l1IssueCount) + comp2 := getComplexityForLevel(constants.IMPACT_LEVEL_2, l2IssueCount) + comp3 := getComplexityForLevel(constants.IMPACT_LEVEL_3, l3IssueCount) complexities := []string{comp1, comp2, comp3} + log.Infof("complexities according to each level: %v", complexities) + finalComplexity := constants.MIGRATION_COMPLEXITY_LOW // If ANY level is HIGH => final is HIGH if slices.Contains(complexities, constants.MIGRATION_COMPLEXITY_HIGH) { - return constants.MIGRATION_COMPLEXITY_HIGH + finalComplexity = constants.MIGRATION_COMPLEXITY_HIGH + } else if slices.Contains(complexities, constants.MIGRATION_COMPLEXITY_MEDIUM) { + // Else if ANY level is MEDIUM => final is MEDIUM + finalComplexity = constants.MIGRATION_COMPLEXITY_MEDIUM } - // Else if ANY level is MEDIUM => final is MEDIUM - if slices.Contains(complexities, constants.MIGRATION_COMPLEXITY_MEDIUM) { - return constants.MIGRATION_COMPLEXITY_MEDIUM - } - return constants.MIGRATION_COMPLEXITY_LOW + + migrationComplexityRationale = buildRationale(finalComplexity, l1IssueCount, l2IssueCount, l3IssueCount) + return finalComplexity } // This is a temporary logic to get migration complexity for oracle based on the migration level from ora2pg report. @@ -200,3 +210,134 @@ func getComplexityForLevel(level string, count int) string { panic(fmt.Sprintf("unknown impact level %s for determining complexity", level)) } } + +// ======================================= Migration Complexity Explanation ========================================== + +// TODO: discuss if the html should be in main report or here +const explainTemplateHTML = ` +{{- if .Summaries }} +

    Below is a breakdown of the issues detected in different categories for each impact level.

    + + + + + + + + + + + + {{- range .Summaries }} + + + + + + + + {{- end }} + +
    CategoryLevel-1Level-2Level-3Total
    {{ .Category }}{{ index .ImpactCounts "LEVEL_1" }}{{ index .ImpactCounts "LEVEL_2" }}{{ index .ImpactCounts "LEVEL_3" }}{{ .TotalIssueCount }}
    +{{- end }} + +

    + Complexity: {{ .Complexity }}
    + Reasoning: {{ .ComplexityRationale }} +

    + +

    +Impact Levels:
    + Level-1: Resolutions are available with minimal effort.
    + Level-2: Resolutions are available requiring moderate effort.
    + Level-3: Resolutions may not be available or are complex. +

    +` + +const explainTemplateText = `Reasoning: {{ .ComplexityRationale }}` + +type MigrationComplexityExplanationData struct { + Summaries []MigrationComplexityCategorySummary + Complexity string + ComplexityRationale string // short reasoning or explanation text +} + +type MigrationComplexityCategorySummary struct { + Category string + TotalIssueCount int + ImpactCounts map[string]int // e.g. {"Level-1": 3, "Level-2": 5, "Level-3": 2} +} + +func buildMigrationComplexityExplanation(sourceDBType string, assessmentReport AssessmentReport, reportFormat string) (string, error) { + if sourceDBType != POSTGRESQL { + return "", nil + } + + var explanation MigrationComplexityExplanationData + explanation.Complexity = assessmentReport.MigrationComplexity + explanation.ComplexityRationale = migrationComplexityRationale + + explanation.Summaries = buildCategorySummary(assessmentReport.Issues) + + var tmpl *template.Template + var err error + if reportFormat == "html" { + tmpl, err = template.New("Explain").Parse(explainTemplateHTML) + } else { + tmpl, err = template.New("Explain").Parse(explainTemplateText) + } + + if err != nil { + return "", fmt.Errorf("failed creating the explanation template: %w", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, explanation); err != nil { + return "", fmt.Errorf("failed executing the template with data: %w", err) + } + return buf.String(), nil +} + +func buildRationale(finalComplexity string, l1Count int, l2Count int, l3Count int) string { + switch finalComplexity { + case constants.MIGRATION_COMPLEXITY_HIGH: + return fmt.Sprintf("Found %d Level-2 issue(s) and %d Level-3 issue(s), resulting in HIGH migration complexity", l2Count, l3Count) + case constants.MIGRATION_COMPLEXITY_MEDIUM: + return fmt.Sprintf("Found %d Level-1 issue(s), %d Level-2 issue(s) and %d Level-3 issue(s), resulting in MEDIUM migration complexity", l1Count, l2Count, l3Count) + case constants.MIGRATION_COMPLEXITY_LOW: + return fmt.Sprintf("Found %d Level-1 issue(s) and %d Level-2 issue(s), resulting in LOW migration complexity", l1Count, l2Count) + } + return "" +} + +func buildCategorySummary(issues []AssessmentIssue) []MigrationComplexityCategorySummary { + if len(issues) == 0 { + return nil + + } + + summaryMap := make(map[string]*MigrationComplexityCategorySummary) + for _, issue := range issues { + if issue.Category == "" { + continue // skipping unknown category issues + } + + if _, ok := summaryMap[issue.Category]; !ok { + summaryMap[issue.Category] = &MigrationComplexityCategorySummary{ + Category: issue.Category, + TotalIssueCount: 0, + ImpactCounts: make(map[string]int), + } + } + + summaryMap[issue.Category].TotalIssueCount++ + summaryMap[issue.Category].ImpactCounts[issue.Impact]++ + } + + var result []MigrationComplexityCategorySummary + for _, summary := range summaryMap { + summary.Category = utils.SnakeCaseToTitleCase(summary.Category) + result = append(result, *summary) + } + return result +} diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 6fe380e1e5..eeef96253b 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -167,6 +167,11 @@ {{ end }} {{end}} + {{if ne .MigrationComplexity "NOT AVAILABLE"}} +

    Migration Complexity Explanation

    +

    {{ .MigrationComplexityExplanation }}

    + {{end}} +

    Unsupported Data Types

    {{.UnsupportedDataTypesDesc}}

    {{ if .UnsupportedDataTypes }} diff --git a/yb-voyager/src/utils/utils.go b/yb-voyager/src/utils/utils.go index b168d886cd..c9967e455b 100644 --- a/yb-voyager/src/utils/utils.go +++ b/yb-voyager/src/utils/utils.go @@ -38,6 +38,8 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) var DoNotPrompt bool @@ -744,3 +746,14 @@ func CheckTools(tools ...string) []string { func BuildObjectName(schemaName, objName string) string { return lo.Ternary(schemaName != "", schemaName+"."+objName, objName) } + +// SnakeCaseToTitleCase converts a snake_case string to a title case string with spaces. +func SnakeCaseToTitleCase(snake string) string { + words := strings.Split(snake, "_") + c := cases.Title(language.English) + for i, word := range words { + words[i] = c.String(word) + } + + return strings.Join(words, " ") +} From 0106500c6a10e442662fc943aa604df4121f5a26 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 13 Jan 2025 11:08:05 +0530 Subject: [PATCH 101/105] release notes for 1.8.8 (#2172) --- RELEASE_NOTES.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d999d808c1..e27854cea2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,31 @@ Included here are the release notes for the [YugabyteDB Voyager](https://docs.yugabyte.com/preview/migrate/) v1 release series. Content will be added as new notable features and changes are available in the patch releases of the YugabyteDB v1 series. +## v1.8.8 - December 24, 2024 + +### Enhancements + +- Assessment and Schema Analysis Reports + - You can now specify the target version of YugabyteDB when running `assess-migration` and `analyze-schema`. Specify the version using the flag `--target-db-version`. The default is the latest stable release (currently 2024.2.0.0). + - Assessment and schema analysis now detect and report the presence of advisory locks, XML functions, and system columns in DDLs. + - Assessment and schema analysis now detect the presence of large objects (and their functions) in DDLs/DMLs. + - In the Schema analysis report (html/text), changed the following field names to improve readability: Invalid Count to Objects with Issues; Total Count to Total Objects; and Valid Count to Objects without Issues. The logic determining when an object is considered to have issues or not has also been improved. + - Stop reporting Unlogged tables as an issue in assessment and schema analysis reports by default, as UNLOGGED no longer results in a syntax error in YugabyteDB v2024.2.0.0. + - Stop reporting ALTER PARTITIONED TABLE ADD PRIMARY KEY as an issue in assessment and schema analysis reports, as the issue has been fixed in YugabyteDB v2024.1.0.0 and later. + - In the assessment report, only statements from `pg_stat_statements` that belong to the schemas provided by the user will be processed for detecting and reporting issues. + +- Data Migration + - `import data file` and `import data to source replica` now accept a new flag `truncate-tables` (in addition to `import data`), which, when used with `start-clean true`, truncates all the tables in the target/source-replica database before importing data into the tables. + +- Miscellaneous + - Enhanced guardrail checks in `import-schema` for YugabyteDB Aeon. + + +### Bug Fixes +- Skip Unsupported Query Constructs detection if `pg_stat_statements` is not loaded via `shared_preloaded_libraries`. +- Prevent Voyager from panicking/erroring out in case of `analyze-schema` and `import data` when `export-dir` is empty. + + ## v1.8.7 - December 10, 2024 ### New Features From 0e1726b2409d8a201d73cc1cc47b437bd2aced98 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Mon, 13 Jan 2025 13:39:52 +0530 Subject: [PATCH 102/105] Update docs links for PG12 and later featurss (#2175) --- .../tests/analyze-schema/expected_issues.json | 107 +++++++++++------- .../expected_schema_analysis_report.json | 16 +-- .../expectedAssessmentReport.json | 22 ++-- .../expectedAssessmentReport.json | 14 ++- yb-voyager/src/query/queryissue/issues_ddl.go | 22 ++-- yb-voyager/src/query/queryissue/issues_dml.go | 48 ++++---- 6 files changed, 132 insertions(+), 97 deletions(-) diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index f702baa5da..8bb85fa16a 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -27,7 +27,8 @@ "Reason": "Json Query Functions", "SqlStatement": "CREATE VIEW public.my_films_view AS\nSELECT jt.* FROM\n my_films,\n JSON_TABLE ( js, '$.favorites[*]'\n COLUMNS (\n id FOR ORDINALITY,\n kind text PATH '$.kind',\n NESTED PATH '$.films[*]' COLUMNS (\n title text FORMAT JSON PATH '$.title' OMIT QUOTES,\n director text PATH '$.director' KEEP QUOTES))) AS jt;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -37,7 +38,8 @@ "Reason": "Json Type Predicate", "SqlStatement": "CREATE TABLE public.json_data (\n id SERIAL PRIMARY KEY,\n data_column TEXT NOT NULL CHECK (data_column IS JSON)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -47,7 +49,8 @@ "Reason": "Json Type Predicate", "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n object_column TEXT CHECK (object_column IS JSON OBJECT)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -57,7 +60,8 @@ "Reason": "Json Type Predicate", "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -67,7 +71,8 @@ "Reason": "Json Type Predicate", "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -77,7 +82,8 @@ "Reason": "Json Type Predicate", "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326),\n array_column TEXT CHECK (array_column IS JSON ARRAY)\n );", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -87,7 +93,8 @@ "Reason": "Json Constructor Functions", "SqlStatement": "CREATE OR REPLACE view test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -97,7 +104,8 @@ "Reason": "Foreign key constraint references partitioned table", "SqlStatement": "CREATE TABLE test_1 (\n\tid numeric NOT NULL REFERENCES sales_data(sales_id),\n\tcountry_code varchar(3),\n\trecord_type varchar(5),\n\tdescriptions varchar(50),\n\tPRIMARY KEY (id)\n) PARTITION BY LIST (country_code, record_type) ;", "Suggestion": "No workaround available ", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -107,7 +115,8 @@ "Reason": "Json Constructor Functions", "SqlStatement": "CREATE MATERIALIZED VIEW test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -117,7 +126,8 @@ "Reason": "Deterministic attribute in collation", "SqlStatement": "CREATE COLLATION special1 (provider = icu, locale = 'en@colCaseFirst=upper;colReorder=grek-latn', deterministic = true);", "Suggestion": "This feature is not supported in YugabyteDB yet", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -127,7 +137,8 @@ "Reason": "Deterministic attribute in collation", "SqlStatement": "CREATE COLLATION ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-true', deterministic = false);", "Suggestion": "This feature is not supported in YugabyteDB yet", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -137,7 +148,7 @@ "Reason": "XML Functions", "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1656,7 +1667,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT pg_advisory_lock(sender_id);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1667,7 +1678,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT pg_advisory_lock(receiver_id);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1678,7 +1689,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT pg_advisory_unlock(sender_id);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1689,7 +1700,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT pg_advisory_unlock(receiver_id);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1700,7 +1711,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT pg_advisory_unlock(sender_id);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1711,7 +1722,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT pg_advisory_unlock(receiver_id);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1722,7 +1733,7 @@ "Reason": "XML Functions", "SqlStatement": "SELECT id, xpath('/person/name/text()', data) AS name FROM test_xml_type;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1733,7 +1744,7 @@ "Reason": "System Columns", "SqlStatement": "SELECT * FROM employees e WHERE e.xmax = (SELECT MAX(xmax) FROM employees WHERE department = e.department);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/24843", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1744,7 +1755,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT id, first_name FROM employees WHERE pg_try_advisory_lock(300) IS TRUE;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1755,7 +1766,7 @@ "Reason": "System Columns", "SqlStatement": "SELECT e.id, e.name,\n ROW_NUMBER() OVER (ORDER BY e.ctid) AS row_num\n FROM employees e;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/24843", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1766,7 +1777,7 @@ "Reason": "XML Functions", "SqlStatement": "SELECT e.id, x.employee_xml\n FROM employees e\n JOIN (\n SELECT xmlelement(name \"employee\", xmlattributes(e.id AS \"id\"), e.name) AS employee_xml\n FROM employees e\n ) x ON x.employee_xml IS NOT NULL\n WHERE xmlexists('//employee[name=\"John Doe\"]' PASSING BY REF x.employee_xml);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1777,7 +1788,7 @@ "Reason": "Advisory Locks", "SqlStatement": "SELECT e.id,\n CASE\n WHEN e.salary \u003e 100000 THEN pg_advisory_lock(e.id)\n ELSE pg_advisory_unlock(e.id)\n END AS lock_status\n FROM employees e;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1788,7 +1799,7 @@ "Reason": "XML Functions", "SqlStatement": "CREATE MATERIALIZED VIEW public.sample_data_view AS\n SELECT sample_data.id,\n sample_data.name,\n sample_data.description,\n XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data,\n pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired,\n sample_data.ctid AS row_ctid,\n sample_data.xmin AS xmin_value\n FROM public.sample_data\n WITH NO DATA;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1799,7 +1810,7 @@ "Reason": "Advisory Locks", "SqlStatement": "CREATE MATERIALIZED VIEW public.sample_data_view AS\n SELECT sample_data.id,\n sample_data.name,\n sample_data.description,\n XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data,\n pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired,\n sample_data.ctid AS row_ctid,\n sample_data.xmin AS xmin_value\n FROM public.sample_data\n WITH NO DATA;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1810,7 +1821,7 @@ "Reason": "System Columns", "SqlStatement": "CREATE MATERIALIZED VIEW public.sample_data_view AS\n SELECT sample_data.id,\n sample_data.name,\n sample_data.description,\n XMLFOREST(sample_data.name AS name, sample_data.description AS description) AS xml_data,\n pg_try_advisory_lock((sample_data.id)::bigint) AS lock_acquired,\n sample_data.ctid AS row_ctid,\n sample_data.xmin AS xmin_value\n FROM public.sample_data\n WITH NO DATA;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/24843", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1821,7 +1832,7 @@ "Reason": "XML Functions", "SqlStatement": "CREATE VIEW public.orders_view AS\n SELECT orders.order_id,\n orders.customer_name,\n orders.product_name,\n orders.quantity,\n orders.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name), XMLELEMENT(NAME \"Quantity\", orders.quantity), XMLELEMENT(NAME \"TotalPrice\", (orders.price * (orders.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,\n orders.ctid AS row_ctid,\n orders.xmin AS transaction_id\n FROM public.orders;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1832,7 +1843,7 @@ "Reason": "Advisory Locks", "SqlStatement": "CREATE VIEW public.orders_view AS\n SELECT orders.order_id,\n orders.customer_name,\n orders.product_name,\n orders.quantity,\n orders.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name), XMLELEMENT(NAME \"Quantity\", orders.quantity), XMLELEMENT(NAME \"TotalPrice\", (orders.price * (orders.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,\n orders.ctid AS row_ctid,\n orders.xmin AS transaction_id\n FROM public.orders;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/3642", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, @@ -1843,7 +1854,7 @@ "Reason": "System Columns", "SqlStatement": "CREATE VIEW public.orders_view AS\n SELECT orders.order_id,\n orders.customer_name,\n orders.product_name,\n orders.quantity,\n orders.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name), XMLELEMENT(NAME \"Quantity\", orders.quantity), XMLELEMENT(NAME \"TotalPrice\", (orders.price * (orders.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", orders.customer_name), XMLELEMENT(NAME \"Product\", orders.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,\n orders.ctid AS row_ctid,\n orders.xmin AS transaction_id\n FROM public.orders;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/24843", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -2008,7 +2019,8 @@ "Reason": "Unsupported datatype - lo on column - raster", "SqlStatement": "CREATE TABLE image (title text, raster lo);", "Suggestion": "Large objects are not yet supported in YugabyteDB, no workaround available currently", - "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#large-objects-and-its-functions-are-currently-not-supported", "MinimumVersionsFixedIn": null }, { @@ -2018,7 +2030,8 @@ "Reason": "FETCH .. WITH TIES", "SqlStatement": "CREATE VIEW top_employees_view AS SELECT * FROM (\n\t\t\tSELECT * FROM employees\n\t\t\tORDER BY salary DESC\n\t\t\tFETCH FIRST 2 ROWS WITH TIES\n\t\t) AS top_employees;", "Suggestion": "No workaround available right now", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2028,7 +2041,8 @@ "Reason": "Unsupported datatype - int8multirange on column - value_ranges", "SqlStatement": "CREATE TABLE bigint_multirange_table (\n id integer PRIMARY KEY,\n value_ranges int8multirange\n);", "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2038,7 +2052,8 @@ "Reason": "Unsupported datatype - datemultirange on column - project_dates", "SqlStatement": "CREATE TABLE date_multirange_table (\n id integer PRIMARY KEY,\n project_dates datemultirange\n);", "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2048,7 +2063,8 @@ "Reason": "Unsupported datatype - int4multirange on column - value_ranges", "SqlStatement": "CREATE TABLE int_multirange_table (\n id integer PRIMARY KEY,\n value_ranges int4multirange\n);", "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2058,7 +2074,8 @@ "Reason": "Unsupported datatype - nummultirange on column - price_ranges", "SqlStatement": "CREATE TABLE numeric_multirange_table (\n id integer PRIMARY KEY,\n price_ranges nummultirange\n);", "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2068,7 +2085,8 @@ "Reason": "Unsupported datatype - tsmultirange on column - event_times", "SqlStatement": "CREATE TABLE timestamp_multirange_table (\n id integer PRIMARY KEY,\n event_times tsmultirange\n);", "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2078,7 +2096,8 @@ "Reason": "Unsupported datatype - tstzmultirange on column - global_event_times", "SqlStatement": "CREATE TABLE timestamptz_multirange_table (\n id integer PRIMARY KEY,\n global_event_times tstzmultirange\n);", "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2088,7 +2107,8 @@ "Reason": "Unique Nulls Not Distinct", "SqlStatement": "CREATE TABLE users_unique_nulls_not_distinct (\n id SERIAL PRIMARY KEY,\n email TEXT,\n UNIQUE NULLS NOT DISTINCT (email)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2098,7 +2118,8 @@ "Reason": "Unique Nulls Not Distinct", "SqlStatement": "CREATE TABLE sales_unique_nulls_not_distinct (\n store_id INT,\n product_id INT,\n sale_date DATE,\n UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date)\n);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2108,7 +2129,8 @@ "Reason": "Unique Nulls Not Distinct", "SqlStatement": "ALTER TABLE sales_unique_nulls_not_distinct_alter\n\tADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -2118,7 +2140,8 @@ "Reason": "Unique Nulls Not Distinct", "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email\n ON users_unique_nulls_not_distinct_index (email)\n NULLS NOT DISTINCT;", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index 40564ee43a..369f9122d9 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -977,7 +977,7 @@ "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidateid,\n businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n modifieddate\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -989,7 +989,7 @@ "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1001,7 +1001,7 @@ "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1013,7 +1013,7 @@ "SqlStatement": "CREATE VIEW person.vadditionalcontactinfo AS\n SELECT p.businessentityid,\n p.firstname,\n p.middlename,\n p.lastname,\n (xpath('(act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS telephonenumber,\n btrim((((xpath('(act:telephoneNumber)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1])::character varying)::text) AS telephonespecialinstructions,\n (xpath('(act:homePostalAddress)[1]/act:Street/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS street,\n (xpath('(act:homePostalAddress)[1]/act:City/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS city,\n (xpath('(act:homePostalAddress)[1]/act:StateProvince/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS stateprovince,\n (xpath('(act:homePostalAddress)[1]/act:PostalCode/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS postalcode,\n (xpath('(act:homePostalAddress)[1]/act:CountryRegion/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS countryregion,\n (xpath('(act:homePostalAddress)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS homeaddressspecialinstructions,\n (xpath('(act:eMail)[1]/act:eMailAddress/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailaddress,\n btrim((((xpath('(act:eMail)[1]/act:SpecialInstructions/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1])::character varying)::text) AS emailspecialinstructions,\n (xpath('((act:eMail)[1]/act:SpecialInstructions/act:telephoneNumber)[1]/act:number/text()'::text, additional.node, '{{act,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactTypes}}'::text[]))[1] AS emailtelephonenumber,\n p.rowguid,\n p.modifieddate\n FROM (person.person p\n LEFT JOIN ( SELECT person.businessentityid,\n unnest(xpath('/ci:AdditionalContactInfo'::text, person.additionalcontactinfo, '{{ci,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ContactInfo}}'::text[])) AS node\n FROM person.person\n WHERE (person.additionalcontactinfo IS NOT NULL)) additional ON ((p.businessentityid = additional.businessentityid)));", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1025,7 +1025,7 @@ "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodelid,\n name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n rowguid,\n modifieddate\n FROM production.productmodel\n WHERE (catalogdescription IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1037,7 +1037,7 @@ "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT productmodelid,\n name,\n ((xpath('/ns:root/text()'::text, instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, step))[1])::character varying(1024) AS \"Step\",\n rowguid,\n modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1049,7 +1049,7 @@ "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (demographics IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null }, @@ -1061,7 +1061,7 @@ "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT businessentityid,\n name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", - "GH": "", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null } diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index 790e5f0cd8..29b451e198 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -99,6 +99,7 @@ "SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n FROM public.employees;" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -109,6 +110,7 @@ "SqlStatement": "CREATE TABLE sales.test_json_chk (\n id integer,\n name text,\n email text,\n active text,\n data jsonb,\n CONSTRAINT test_json_chk_data_check CHECK ((data['key'::text] \u003c\u003e '{}'::jsonb))\n);" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", "MinimumVersionsFixedIn": null }, { @@ -119,6 +121,7 @@ "SqlStatement": "CREATE TABLE sales.json_data (\n id integer NOT NULL,\n array_column text,\n unique_keys_column text,\n CONSTRAINT json_data_array_column_check CHECK ((array_column IS JSON ARRAY)),\n CONSTRAINT json_data_unique_keys_column_check CHECK ((unique_keys_column IS JSON WITH UNIQUE KEYS))\n);" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null } ], @@ -235,56 +238,56 @@ { "ConstructTypeName": "Aggregate Functions", "Query": "SELECT range_intersect_agg(event_range) AS intersection_of_ranges\nFROM sales.events", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Aggregate Functions", "Query": "SELECT range_agg(event_range) AS union_of_ranges\nFROM sales.events", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Aggregate Functions", "Query": "SELECT\n any_value(name) AS any_employee\n FROM employees", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Merge Statement", "Query": "MERGE INTO sales.customer_account ca\nUSING sales.recent_transactions t \nON t.customer_id = ca.customer_id\nWHEN MATCHED THEN\n UPDATE SET balance = balance + transaction_value\nWHEN NOT MATCHED THEN\n INSERT (customer_id, balance)\n VALUES (t.customer_id, t.transaction_value)", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#merge-command", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Jsonb Subscripting", "Query": "SELECT \n data,\n data[$1] AS name, \n (data[$2]) as active\nFROM sales.test_json_chk", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Jsonb Subscripting", "Query": "SELECT (sales.get_user_info($1))[$2] AS user_info", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Jsonb Subscripting", "Query": "SELECT (jsonb_build_object($1, $2, $3, $4, $5, $6) || $7)[$8] AS json_obj", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Jsonb Subscripting", "Query": "SELECT ($1 :: jsonb)[$2][$3] as b", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "Json Type Predicate", "Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null } ], @@ -298,6 +301,7 @@ "SqlStatement": "SELECT\n data,\n data['name'] AS name,\n (data['active']) as active\n FROM sales.test_json_chk;" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 58c8596964..d26b9969e8 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -597,6 +597,7 @@ "SqlStatement": "CREATE COLLATION schema2.ignore_accents (provider = icu, deterministic = false, locale = 'und-u-kc-ks-level1');" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -659,6 +660,7 @@ "SqlStatement": "CREATE TRIGGER t_raster BEFORE DELETE OR UPDATE ON public.combined_tbl FOR EACH ROW EXECUTE FUNCTION public.lo_manage('raster');" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#large-objects-and-its-functions-are-currently-not-supported", "MinimumVersionsFixedIn": null }, { @@ -669,6 +671,7 @@ "SqlStatement": "ALTER TABLE ONLY public.test_jsonb\n ADD CONSTRAINT test_jsonb_id_region_fkey FOREIGN KEY (id, region) REFERENCES public.sales_region(id, region);" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -679,6 +682,7 @@ "SqlStatement": "CREATE TABLE public.ordersentry (\n order_id integer NOT NULL,\n customer_name text NOT NULL,\n product_name text NOT NULL,\n quantity integer NOT NULL,\n price numeric(10,2) NOT NULL,\n processed_at timestamp without time zone,\n r integer DEFAULT regexp_count('This is an example. Another example. Example is a common word.'::text, 'example'::text)\n);" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -693,6 +697,7 @@ "SqlStatement": "CREATE VIEW schema2.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM schema2.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -703,6 +708,7 @@ "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employee_id,\n first_name\n FROM public.employees;" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { @@ -741,6 +747,7 @@ "SqlStatement": "ALTER TABLE ONLY schema2.users_unique_nulls_not_distinct\n ADD CONSTRAINT users_unique_nulls_not_distinct_email_key UNIQUE NULLS NOT DISTINCT (email);" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null } ], @@ -2861,19 +2868,19 @@ { "ConstructTypeName": "Large Object Functions", "Query": "SELECT lo_create($1)", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#large-objects-and-its-functions-are-currently-not-supported", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "COPY FROM ... WHERE", "Query": "COPY employeesCopyFromWhere (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, { "ConstructTypeName": "COPY ... ON_ERROR", "Query": "COPY employeesCopyOnError (id, name, age)\nFROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE )", - "DocsLink": "", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null } ], @@ -2887,6 +2894,7 @@ "SqlStatement": "SELECT lo_unlink(loid);" } ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#large-objects-and-its-functions-are-currently-not-supported", "MinimumVersionsFixedIn": null }, { diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 468ef2bc5c..6f3da748cd 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -463,7 +463,7 @@ var loDatatypeIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_1, Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", - DocsLink: "", // TODO + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#large-objects-and-its-functions-are-currently-not-supported", // TODO } func NewLODatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { @@ -477,8 +477,8 @@ var multiRangeDatatypeIssue = issue.Issue{ Name: "Unsupported datatype", Impact: constants.IMPACT_LEVEL_1, Suggestion: "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", - GH: "", //TODO - DocsLink: "", //TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewMultiRangeDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { @@ -492,8 +492,8 @@ var securityInvokerViewIssue = issue.Issue{ Name: "Security Invoker Views not supported yet", Impact: constants.IMPACT_LEVEL_1, Suggestion: "Security Invoker Views are not yet supported in YugabyteDB, no workaround available currently", - GH: "", // TODO - DocsLink: "", // TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatement string) QueryIssue { @@ -505,8 +505,8 @@ var deterministicOptionCollationIssue = issue.Issue{ Name: DETERMINISTIC_OPTION_WITH_COLLATION_NAME, Impact: constants.IMPACT_LEVEL_1, Suggestion: "This feature is not supported in YugabyteDB yet", - GH: "", // TODO - DocsLink: "", // TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewDeterministicOptionCollationIssue(objectType string, objectName string, SqlStatement string) QueryIssue { @@ -518,8 +518,8 @@ var foreignKeyReferencesPartitionedTableIssue = issue.Issue{ Name: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, Impact: constants.IMPACT_LEVEL_1, Suggestion: "No workaround available ", - GH: "", // TODO - DocsLink: "", // TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewForeignKeyReferencesPartitionedTableIssue(objectType string, objectName string, SqlStatement string, constraintName string) QueryIssue { @@ -534,8 +534,8 @@ var uniqueNullsNotDistinctIssue = issue.Issue{ Name: UNIQUE_NULLS_NOT_DISTINCT_NAME, Impact: constants.IMPACT_LEVEL_1, Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewUniqueNullsNotDistinctIssue(objectType string, objectName string, sqlStatement string) QueryIssue { diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 322863f4a3..8bc2c356cc 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -29,7 +29,7 @@ var advisoryLocksIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", - GH: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/3642", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", } @@ -43,7 +43,7 @@ var systemColumnsIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", - GH: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/24843", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", } @@ -57,7 +57,7 @@ var xmlFunctionsIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", - GH: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/1043", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", } @@ -71,8 +71,8 @@ var regexFunctionsIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -85,8 +85,8 @@ var aggregateFunctionIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -103,8 +103,8 @@ var jsonConstructorFunctionsIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "Postgresql 17 features not supported yet in YugabyteDB", Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -121,8 +121,8 @@ var jsonQueryFunctionIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "Postgresql 17 features not supported yet in YugabyteDB", Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -140,7 +140,7 @@ var loFunctionsIssue = issue.Issue{ Description: "Large Objects functions are not supported in YugabyteDB", Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", - DocsLink: "", //TODO + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#large-objects-and-its-functions-are-currently-not-supported", } func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -157,8 +157,8 @@ var jsonbSubscriptingIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "Jsonb subscripting is not supported in YugabyteDB yet", Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", - GH: "", - DocsLink: "", //TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#jsonb-subscripting", } func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -171,8 +171,8 @@ var jsonPredicateIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "IS JSON predicate expressions not supported yet in YugabyteDB", Suggestion: "", - GH: "", - DocsLink: "", //TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -185,8 +185,8 @@ var copyFromWhereIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -199,8 +199,8 @@ var copyOnErrorIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", - GH: "", - DocsLink: "", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -213,8 +213,8 @@ var fetchWithTiesIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "FETCH .. WITH TIES is not supported in YugabyteDB", Suggestion: "No workaround available right now", - GH: "", - DocsLink: "", //TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", } func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -227,8 +227,8 @@ var mergeStatementIssue = issue.Issue{ Impact: constants.IMPACT_LEVEL_2, Description: "This statement is not supported in YugabyteDB yet", Suggestion: "Use PL/pgSQL to write the logic to get this functionality", - GH: "", - DocsLink: "", //TODO + GH: "https://github.com/yugabyte/yugabyte-db/issues/25574", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#merge-command", } func NewMergeStatementIssue(objectType string, objectName string, sqlStatement string) QueryIssue { From a70ef0a75316e2dcd4c154faef06402ab6d8862d Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 13 Jan 2025 20:49:19 +0530 Subject: [PATCH 103/105] setting error as empty string in callhome (#2178) --- yb-voyager/src/callhome/diagnostics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 9f4d40e7bd..f8dcc71652 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -24,7 +24,6 @@ import ( "os" "reflect" "strconv" - "strings" "github.com/google/uuid" "github.com/samber/lo" @@ -298,5 +297,6 @@ func SendPayload(payload *Payload) error { // Note: This is a temporary solution. A better solution would be to have // properly structured errors and only send the generic error message to callhome. func SanitizeErrorMsg(errorMsg string) string { - return strings.Split(errorMsg, ":")[0] + return "" // For now, returning empty string. After thorough testing, we can return the specific error message. + // return strings.Split(errorMsg, ":")[0] } From a5bd25afdd157e4545660b5afeca78585dd5fcfc Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 15 Jan 2025 15:23:01 +0530 Subject: [PATCH 104/105] Added Release notes for voyager 1.8.9 release (#2185) --- RELEASE_NOTES.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e27854cea2..dccd04631e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,33 @@ Included here are the release notes for the [YugabyteDB Voyager](https://docs.yugabyte.com/preview/migrate/) v1 release series. Content will be added as new notable features and changes are available in the patch releases of the YugabyteDB v1 series. +## v1.8.9 - January 14, 2025 + +### New Features +- Implemented a new algorithm for migration complexity determination that accounts for all potential issues, including unsupported query constructs, PL/pgSQL objects, and incompatible data types. +- Introduced migration complexity explanations in the PostgreSQL assessment report, summarising high-impact issues and illustrating how the overall complexity level is determined. + +### Enhancements +- Enhanced Assessment and Schema Analysis reports to detect unsupported PostgreSQL features from PG 12 up to PG 17, including: + - Regexp functions (`regexp_count`, `regexp_instr`, `regexp_like`) + - Security Invoker Views + - JSON **constructor** and JSON **Query functions** + - IS_JSON predicate clauses(`IS_JSON`, `IS JSON SCALAR`, `IS JSON OBJECT`, `IS JSON ARRAY`) + - Aggregate functions like `anyvalue`, `range_agg`, `range_intersect_agg` + - COPY command syntax such as `COPY FROM ... WHERE` and `COPY ... ON_ERROR` + - **Multirange datatypes** like `int4multirange`, `int8multirange`, `datemultirange` etc.. + - `FETCH FIRST … WITH TIES` subclause in `SELECT` statement + - Foreign Key referencing a partitioned table + - JSONB Subscripting in DML, DDL or PL/PGSQL + - `UNIQUE NULLS NOT DISTINCT` in `CREATE/ALTER TABLE` statement + - The **deterministic** attribute in `CREATE COLLATION` + - `MERGE` statements + +### Bug Fixes +- Fixed an [issue](https://github.com/yugabyte/yb-voyager/issues/2034) where import data failed for tables whose datafile paths exceeded 250 characters. The fix is backward compatible, allowing migrations started with older voyager version to continue seamlessly. +- Fixed an issue where logic for detecting unsupported PostgreSQL versions was giving false positives. + + ## v1.8.8 - December 24, 2024 ### Enhancements From 8b918601bf8ed62c8b40be913572281666bfd460 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 15 Jan 2025 20:12:33 +0530 Subject: [PATCH 105/105] Fix: retrieving Migration UUID as early as possible in the commands (#2162) --- yb-voyager/cmd/analyzeSchema.go | 10 +++++----- yb-voyager/cmd/assessMigrationBulkCommand.go | 11 +++++------ yb-voyager/cmd/assessMigrationCommand.go | 14 ++++++-------- yb-voyager/cmd/endMigrationCommand.go | 14 +++++++++----- yb-voyager/cmd/exportData.go | 10 +++++----- yb-voyager/cmd/importData.go | 11 +++++------ yb-voyager/cmd/importDataFileCommand.go | 11 +++++------ yb-voyager/cmd/importSchema.go | 12 ++++++------ yb-voyager/src/callhome/diagnostics.go | 2 +- 9 files changed, 47 insertions(+), 48 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 8ff11381a5..fe25825e65 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1125,10 +1125,6 @@ func checkConversions(sqlInfoArr []sqlInfo, filePath string) { } func analyzeSchema() { - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } utils.PrintAndLog("Analyzing schema for target YugabyteDB version %s\n", targetDbVersion) schemaAnalysisStartedEvent := createSchemaAnalysisStartedEvent() @@ -1290,9 +1286,13 @@ var analyzeSchemaCmd = &cobra.Command{ "For more details and examples, visit https://docs.yugabyte.com/preview/yugabyte-voyager/reference/schema-migration/analyze-schema/", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } validOutputFormats := []string{"html", "json", "txt", "xml"} validateReportOutputFormat(validOutputFormats, analyzeSchemaReportFormat) - err := validateAndSetTargetDbVersionFlag() + err = validateAndSetTargetDbVersionFlag() if err != nil { utils.ErrExit("%v", err) } diff --git a/yb-voyager/cmd/assessMigrationBulkCommand.go b/yb-voyager/cmd/assessMigrationBulkCommand.go index d758c2e922..a4973a7e93 100644 --- a/yb-voyager/cmd/assessMigrationBulkCommand.go +++ b/yb-voyager/cmd/assessMigrationBulkCommand.go @@ -48,7 +48,11 @@ var assessMigrationBulkCmd = &cobra.Command{ Long: "Bulk Assessment of multiple schemas across one or more Oracle database instances", PreRun: func(cmd *cobra.Command, args []string) { - err := validateFleetConfigFile(fleetConfigPath) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + err = validateFleetConfigFile(fleetConfigPath) if err != nil { utils.ErrExit("validating fleet config file: %s", err.Error()) } @@ -156,11 +160,6 @@ func assessMigrationBulk() error { return fmt.Errorf("failed to parse fleet config file: %w", err) } - err = retrieveMigrationUUID() - if err != nil { - return fmt.Errorf("failed to get migration UUID: %w", err) - } - for _, dbConfig := range bulkAssessmentDBConfigs { utils.PrintAndLog("\nAssessing '%s' schema", dbConfig.GetSchemaIdentifier()) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 586610676f..e2636fd57e 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -80,13 +80,18 @@ var assessMigrationCmd = &cobra.Command{ Long: fmt.Sprintf("Assess the migration from source (%s) database to YugabyteDB.", strings.Join(assessMigrationSupportedDBTypes, ", ")), PreRun: func(cmd *cobra.Command, args []string) { + CreateMigrationProjectIfNotExists(source.DBType, exportDir) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } validateSourceDBTypeForAssessMigration() setExportFlagsDefaults() validateSourceSchema() validatePortRange() validateSSLMode() validateOracleParams() - err := validateAndSetTargetDbVersionFlag() + err = validateAndSetTargetDbVersionFlag() if err != nil { utils.ErrExit("%v", err) } @@ -318,13 +323,6 @@ func assessMigration() (err error) { schemaDir = filepath.Join(assessmentMetadataDir, "schema") checkStartCleanForAssessMigration(assessmentMetadataDirFlag != "") - CreateMigrationProjectIfNotExists(source.DBType, exportDir) - - err = retrieveMigrationUUID() - if err != nil { - return fmt.Errorf("failed to get migration UUID: %w", err) - } - utils.PrintAndLog("Assessing for migration to target YugabyteDB version %s\n", targetDbVersion) assessmentDir := filepath.Join(exportDir, "assessment") diff --git a/yb-voyager/cmd/endMigrationCommand.go b/yb-voyager/cmd/endMigrationCommand.go index c161d79a87..61ef467ce0 100644 --- a/yb-voyager/cmd/endMigrationCommand.go +++ b/yb-voyager/cmd/endMigrationCommand.go @@ -43,14 +43,19 @@ var endMigrationCmd = &cobra.Command{ Long: "End the current migration and cleanup all metadata stored in databases(Target, Source-Replica and Source) and export-dir", PreRun: func(cmd *cobra.Command, args []string) { - err := validateEndMigrationFlags(cmd) + if utils.IsDirectoryEmpty(exportDir) { + utils.ErrExit("export directory is empty, nothing to end") + } + + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + err = validateEndMigrationFlags(cmd) if err != nil { utils.ErrExit(err.Error()) } - if utils.IsDirectoryEmpty(exportDir) { - utils.ErrExit("export directory is empty, nothing to end") - } }, Run: endMigrationCommandFn, @@ -76,7 +81,6 @@ func endMigrationCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("error while checking streaming mode: %w\n", err) } - retrieveMigrationUUID() checkIfEndCommandCanBePerformed(msr) // backing up the state from the export directory diff --git a/yb-voyager/cmd/exportData.go b/yb-voyager/cmd/exportData.go index 0620700223..bd9a37f900 100644 --- a/yb-voyager/cmd/exportData.go +++ b/yb-voyager/cmd/exportData.go @@ -112,6 +112,11 @@ func exportDataCommandPreRun(cmd *cobra.Command, args []string) { func exportDataCommandFn(cmd *cobra.Command, args []string) { CreateMigrationProjectIfNotExists(source.DBType, exportDir) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + ExitIfAlreadyCutover(exporterRole) if useDebezium && !changeStreamingIsEnabled(exportType) { utils.PrintAndLog("Note: Beta feature to accelerate data export is enabled by setting BETA_FAST_DATA_EXPORT environment variable") @@ -122,11 +127,6 @@ func exportDataCommandFn(cmd *cobra.Command, args []string) { utils.PrintAndLog("export of data for source type as '%s'", source.DBType) sqlname.SourceDBType = source.DBType - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } - success := exportData() if success { sendPayloadAsPerExporterRole(COMPLETE, "") diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 45050cf615..29ce170ec7 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -80,8 +80,12 @@ var importDataCmd = &cobra.Command{ if importerRole == "" { importerRole = TARGET_DB_IMPORTER_ROLE } + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } sourceDBType = GetSourceDBTypeFromMSR() - err := validateImportFlags(cmd, importerRole) + err = validateImportFlags(cmd, importerRole) if err != nil { utils.ErrExit("Error: %s", err.Error()) } @@ -132,11 +136,6 @@ func importDataCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("Failed to get migration status record: %s", err) } - err = retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } - // Check if target DB has the required permissions if tconf.RunGuardrailsChecks { checkImportDataPermissions() diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 340d7ef057..1e6aa10863 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -72,10 +72,13 @@ var importDataFileCmd = &cobra.Command{ sourceDBType = POSTGRESQL // dummy value - this command is not affected by it sqlname.SourceDBType = sourceDBType CreateMigrationProjectIfNotExists(sourceDBType, exportDir) - + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } tconf.Schema = strings.ToLower(tconf.Schema) tdb = tgtdb.NewTargetDB(&tconf) - err := tdb.Init() + err = tdb.Init() if err != nil { utils.ErrExit("Failed to initialize the target DB: %s", err) } @@ -91,10 +94,6 @@ var importDataFileCmd = &cobra.Command{ dataStore = datastore.NewDataStore(dataDir) importFileTasks := prepareImportFileTasks() prepareForImportDataCmd(importFileTasks) - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } importData(importFileTasks) packAndSendImportDataFilePayload(COMPLETE, "") diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index 367ac0002a..c26d3b89f4 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -47,8 +47,12 @@ var importSchemaCmd = &cobra.Command{ if tconf.TargetDBType == "" { tconf.TargetDBType = YUGABYTEDB } + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } sourceDBType = GetSourceDBTypeFromMSR() - err := validateImportFlags(cmd, TARGET_DB_IMPORTER_ROLE) + err = validateImportFlags(cmd, TARGET_DB_IMPORTER_ROLE) if err != nil { utils.ErrExit("Error: %s", err.Error()) } @@ -79,10 +83,6 @@ var flagRefreshMViews utils.BoolStr var invalidTargetIndexesCache map[string]bool func importSchema() error { - err := retrieveMigrationUUID() - if err != nil { - return fmt.Errorf("failed to get migration UUID: %w", err) - } tconf.Schema = strings.ToLower(tconf.Schema) @@ -93,7 +93,7 @@ func importSchema() error { // available always and this is just for initialisation of tdb and marking it nil again back. tconf.Schema = "public" tdb = tgtdb.NewTargetDB(&tconf) - err = tdb.Init() + err := tdb.Init() if err != nil { utils.ErrExit("Failed to initialize the target DB: %s", err) } diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index f8dcc71652..7bf14d6d8b 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -58,7 +58,7 @@ CREATE TABLE diagnostics ( migration_type TEXT, time_taken_sec int, status TEXT, - host_ip character varying 255 -- set by the callhome service + host_ip character varying (255), -- set in callhome service PRIMARY KEY (migration_uuid, migration_phase, collected_at) );