Merge branch 'main' into priyanshi/fix-hstore

.github/workflows/misc-migtests.yml

@@ -114,9 +114,6 @@ jobs:
             echo "127.0.0.1    yb-master-n1" | sudo tee -a /etc/hosts
             psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
 
-            echo "TEST: PG sample schemas (omnibus)"
-            migtests/scripts/run-schema-migration.sh pg/omnibus
-
             echo "Setup the gcp credentials"
             migtests/scripts/gcs/create_gcs_credentials_file
 

.github/workflows/mysql-migtests.yml

@@ -137,6 +137,7 @@ jobs:
 
       # Placeholder for now so that a basic test can run
       - name: Create the live migration user
+        if: always()
         run: |
           mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';'
 

.github/workflows/pg-17-migtests.yml

@@ -117,6 +117,10 @@ jobs:
         if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-schema-migration.sh pg/osm
 
+      - name: "TEST: PG sample schemas (omnibus)"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-schema-migration.sh pg/omnibus
+
       - name: "TEST: PG sample schemas (adventureworks)"
         if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-schema-migration.sh pg/adventureworks

RELEASE_NOTES.md

@@ -4,6 +4,58 @@
 
 Included here are the release notes for the [YugabyteDB Voyager](https://docs.yugabyte.com/preview/migrate/) v1 release series. Content will be added as new notable features and changes are available in the patch releases of the YugabyteDB v1 series.
 
+## v1.8.9 - January 14, 2025
+
+### New Features
+- Implemented a new algorithm for migration complexity determination that accounts for all potential issues, including unsupported query constructs, PL/pgSQL objects, and incompatible data types.
+- Introduced migration complexity explanations in the PostgreSQL assessment report, summarising high-impact issues and illustrating how the overall complexity level is determined.
+
+### Enhancements
+- Enhanced Assessment and Schema Analysis reports to detect unsupported PostgreSQL features from PG 12 up to PG 17, including:
+  - Regexp functions (`regexp_count`, `regexp_instr`, `regexp_like`)
+  - Security Invoker Views
+  - JSON **constructor** and JSON **Query functions**
+  - IS_JSON predicate clauses(`IS_JSON`, `IS JSON SCALAR`, `IS JSON OBJECT`, `IS JSON ARRAY`)
+  - Aggregate functions like `anyvalue`, `range_agg`, `range_intersect_agg`
+  - COPY command syntax such as `COPY FROM ... WHERE` and `COPY ... ON_ERROR`
+  - **Multirange datatypes** like `int4multirange`, `int8multirange`, `datemultirange` etc..
+  - `FETCH FIRST … WITH TIES` subclause in `SELECT` statement
+  - Foreign Key referencing a partitioned table
+  - JSONB Subscripting in DML, DDL or PL/PGSQL
+  - `UNIQUE NULLS NOT DISTINCT` in `CREATE/ALTER TABLE` statement
+  - The **deterministic** attribute in `CREATE COLLATION`
+  - `MERGE` statements 
+
+### Bug Fixes
+- Fixed an [issue](https://github.com/yugabyte/yb-voyager/issues/2034) where import data failed for tables whose datafile paths exceeded 250 characters. The fix is backward compatible, allowing migrations started with older voyager version to continue seamlessly.
+- Fixed an issue where logic for detecting unsupported PostgreSQL versions was giving false positives.
+
+
+## v1.8.8 - December 24, 2024
+
+### Enhancements
+
+- Assessment and Schema Analysis Reports
+  - You can now specify the target version of YugabyteDB when running `assess-migration` and `analyze-schema`. Specify the version using the flag `--target-db-version`. The default is the latest stable release (currently 2024.2.0.0).
+  - Assessment and schema analysis now detect and report the presence of advisory locks, XML functions, and system columns in DDLs.
+  - Assessment and schema analysis now detect the presence of large objects (and their functions) in DDLs/DMLs.
+  - In the Schema analysis report (html/text), changed the following field names to improve readability: Invalid Count to Objects with Issues; Total Count to Total Objects; and Valid Count to Objects without Issues. The logic determining when an object is considered to have issues or not has also been improved.
+  - Stop reporting Unlogged tables as an issue in assessment and schema analysis reports by default, as UNLOGGED no longer results in a syntax error in YugabyteDB v2024.2.0.0.
+  - Stop reporting ALTER PARTITIONED TABLE ADD PRIMARY KEY as an issue in assessment and schema analysis reports, as the issue has been fixed in YugabyteDB v2024.1.0.0 and later.
+  - In the assessment report, only statements from `pg_stat_statements` that belong to the schemas provided by the user will be processed for detecting and reporting issues.
+
+- Data Migration
+  - `import data file` and `import data to source replica` now accept a new flag `truncate-tables` (in addition to `import data`), which, when used with `start-clean true`, truncates all the tables in the target/source-replica database before importing data into the tables.
+
+- Miscellaneous
+  - Enhanced guardrail checks in `import-schema` for YugabyteDB Aeon.
+
+
+### Bug Fixes
+- Skip Unsupported Query Constructs detection if `pg_stat_statements` is not loaded via `shared_preloaded_libraries`.
+- Prevent Voyager from panicking/erroring out in case of `analyze-schema` and `import data` when `export-dir` is empty.
+
+
 ## v1.8.7 - December 10, 2024
 
 ### New Features

migtests/lib/yb.py

@@ -52,7 +52,8 @@ def new_source_db():
 
 def verify_colocation(tgt, source_db_type):
 	print("Verifying the colocation of the tables")
-	json_file = "export-dir/assessment/reports/migration_assessment_report.json"
+	export_dir = os.getenv("EXPORT_DIR", "export-dir")
+	json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json"
 
 	sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file)
 

migtests/scripts/add-pk-from-alter-to-create

@@ -5,7 +5,7 @@ import os
 import re
 import shutil
 
-table_file_path = os.getenv('TEST_DIR')+'/export-dir/schema/tables/table.sql'
+table_file_path = os.getenv('EXPORT_DIR')+'/schema/tables/table.sql'
 
 #copy the table_file_path to table_file_path+'.old'
 src = table_file_path

migtests/scripts/functions.sh

@@ -134,7 +134,8 @@ grant_user_permission_oracle(){
 	*/
 	GRANT FLASHBACK ANY TABLE TO ybvoyager;
 EOF
-	run_sqlplus_as_sys ${db_name} "oracle-inputs.sql"	
+	run_sqlplus_as_sys ${db_name} "oracle-inputs.sql"
+	rm oracle-inputs.sql
 
 }
 
@@ -149,6 +150,8 @@ EOF
 	run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql"
 	cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql
 	run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql"
+	rm create-pdb-tablespace.sql
+	rm oracle-inputs.sql
 }
 
 grant_permissions_for_live_migration_pg() {
@@ -191,7 +194,7 @@ run_sqlplus_as_sys() {
 run_sqlplus_as_schema_owner() {
     db_name=$1
     sql=$2
-    conn_string="${SOURCE_DB_USER_SCHEMA_OWNER}/${SOURCE_DB_USER_SCHEMA_OWNER_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}"
+    conn_string="${SOURCE_DB_SCHEMA}/${SOURCE_DB_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}"
     echo exit | sqlplus -f "${conn_string}" @"${sql}"
 }
 
@@ -623,16 +626,6 @@ get_value_from_msr(){
   echo $val
 }
 
-create_ff_schema(){
-	db_name=$1
-
-	cat > create-ff-schema.sql << EOF
-	CREATE USER FF_SCHEMA IDENTIFIED BY "password";
-	GRANT all privileges to FF_SCHEMA;
-EOF
-	run_sqlplus_as_sys ${db_name} "create-ff-schema.sql"
-}
-
 set_replica_identity(){
 	db_schema=$1
     cat > alter_replica_identity.sql <<EOF
@@ -647,6 +640,7 @@ set_replica_identity(){
     END \$CUSTOM\$;
 EOF
     run_psql ${SOURCE_DB_NAME} "$(cat alter_replica_identity.sql)"
+	rm alter_replica_identity.sql
 }
 
 grant_permissions_for_live_migration() {
@@ -673,7 +667,15 @@ grant_permissions_for_live_migration() {
 setup_fallback_environment() {
 	if [ "${SOURCE_DB_TYPE}" = "oracle" ]; then
 		run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql
-		run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql
+
+		TEMP_SCRIPT="/tmp/fall_back_prep.sql"
+
+		sed "s/TEST_SCHEMA/${SOURCE_DB_SCHEMA}/g" ${SCRIPTS}/oracle/fall_back_prep.sql > $TEMP_SCRIPT
+
+		run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT
+
+		# Clean up the temporary file after execution
+		rm -f $TEMP_SCRIPT
 		elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then
 		conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${SOURCE_DB_NAME}"
 		psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql
@@ -877,18 +879,32 @@ normalize_json() {
     # Normalize JSON with jq; use --sort-keys to avoid the need to keep the same sequence of keys in expected vs actual json
     jq --sort-keys 'walk(
         if type == "object" then
-            .ObjectNames? |= (if type == "string" then split(", ") | sort | join(", ") else . end) |
+            .ObjectNames? |= (
+				if type == "string" then
+					split(", ") | sort | join(", ")
+				else
+					.
+				end
+			) |
             .VoyagerVersion? = "IGNORED" |
 			.TargetDBVersion? = "IGNORED" |
             .DbVersion? = "IGNORED" |
             .FilePath? = "IGNORED" |
             .OptimalSelectConnectionsPerNode? = "IGNORED" |
             .OptimalInsertConnectionsPerNode? = "IGNORED" |
             .RowCount? = "IGNORED" |
-            .SqlStatement? |= (if type == "string" then gsub("\\n"; " ") else . end)
+			.MigrationComplexityExplanation?= "IGNORED" |
+            # Replace newline characters in SqlStatement with spaces
+			.SqlStatement? |= (
+				if type == "string" then
+					gsub("\\n"; " ")
+				else
+					.
+				end
+			)
         elif type == "array" then
-            sort_by(tostring)
-        else
+			sort_by(tostring)
+		else
             .
         end
     )' "$input_file" > "$temp_file"
@@ -1044,3 +1060,60 @@ cutover_to_target() {
 
     yb-voyager initiate cutover to target ${args} $*
 }
+
+create_source_db() {
+	source_db=$1
+	case ${SOURCE_DB_TYPE} in
+		postgresql)
+			run_psql postgres "DROP DATABASE IF EXISTS ${source_db};"
+			run_psql postgres "CREATE DATABASE ${source_db};"
+			;;
+		mysql)
+			run_mysql mysql "DROP DATABASE IF EXISTS ${source_db};"
+			run_mysql mysql "CREATE DATABASE ${source_db};"
+			;;
+		oracle)
+			cat > create-oracle-schema.sql << EOF
+			CREATE USER ${source_db} IDENTIFIED BY "password";
+			GRANT all privileges to ${source_db};
+EOF
+			run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-oracle-schema.sql"
+			rm create-oracle-schema.sql
+			;;
+		*)
+			echo "ERROR: Source DB not created for ${SOURCE_DB_TYPE}"
+			exit 1
+			;;
+	esac
+}
+
+normalize_and_export_vars() {
+    local test_suffix=$1
+
+    # Normalize TEST_NAME
+	# Keeping the full name for PG and MySQL to test out large schema/export dir names
+    export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')"
+
+    # Set EXPORT_DIR
+    export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"}
+    if [ -n "${SOURCE_DB_SSL_MODE}" ]; then
+        EXPORT_DIR="${EXPORT_DIR}_ssl"
+    fi
+
+    # Set database-specific variables
+    case "${SOURCE_DB_TYPE}" in
+        postgresql|mysql)
+            export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"}
+            ;;
+        oracle)
+            # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit
+			# Since test_suffix is the unique identifying factor, we need to add it post all the normalization 
+            export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"}
+            export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^}
+            ;;
+        *)
+            echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}"
+            exit 1
+            ;;
+    esac
+}

migtests/scripts/live-migration-fallb-run-test.sh

@@ -18,9 +18,7 @@ export REPO_ROOT="${PWD}"
 export SCRIPTS="${REPO_ROOT}/migtests/scripts"
 export TESTS_DIR="${REPO_ROOT}/migtests/tests"
 export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
-export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
 export QUEUE_SEGMENT_MAX_BYTES=400
-
 export PYTHONPATH="${REPO_ROOT}/migtests/lib"
 
 # Order of env.sh import matters.
@@ -37,10 +35,12 @@ else
 	source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
 fi
 
-source ${SCRIPTS}/yugabytedb/env.sh
-
 source ${SCRIPTS}/functions.sh
 
+normalize_and_export_vars "fallb"
+
+source ${SCRIPTS}/yugabytedb/env.sh
+
 main() {
 
 	echo "Deleting the parent export-dir present in the test directory"
@@ -56,6 +56,10 @@ main() {
 	pushd ${TEST_DIR}
 
 	step "Initialise source database."
+	if [ "${SOURCE_DB_TYPE}" = "oracle" ]
+	then
+		create_source_db ${SOURCE_DB_SCHEMA}
+	fi
 	./init-db
 
 	step "Grant source database user permissions for live migration"	
@@ -169,7 +173,11 @@ main() {
 	import_schema --post-snapshot-import true --refresh-mviews=true
 
 	step "Run snapshot validations."
-	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true'
+	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' || {
+			tail_log_file "yb-voyager-import-data.log"
+			tail_log_file "yb-voyager-export-data-from-source.log"
+			exit 1
+		} 
 
 	step "Inserting new events"
 	run_sql_file source_delta.sql
@@ -264,7 +272,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/live-migration-fallf-run-test.sh

@@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}"
 export SCRIPTS="${REPO_ROOT}/migtests/scripts"
 export TESTS_DIR="${REPO_ROOT}/migtests/tests"
 export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
-export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
 
 export PYTHONPATH="${REPO_ROOT}/migtests/lib"
 export PATH="${PATH}:/usr/lib/oracle/21/client64/bin"
@@ -38,12 +37,14 @@ else
     source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
 fi
 
+source ${SCRIPTS}/functions.sh
+
+normalize_and_export_vars "fallf"
+
 source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh
 
 source ${SCRIPTS}/yugabytedb/env.sh
 
-source ${SCRIPTS}/functions.sh
-
 main() {
 
 	echo "Deleting the parent export-dir present in the test directory"
@@ -62,7 +63,9 @@ main() {
 
 	if [ "${SOURCE_DB_TYPE}" = "oracle" ]
 	then
-		create_ff_schema ${SOURCE_REPLICA_DB_NAME}
+		create_source_db ${SOURCE_DB_SCHEMA}
+		# TODO: Add dynamic Fall Forward schema creation. Currently using the same name for all tests.
+		create_source_db ${SOURCE_REPLICA_DB_SCHEMA}
 		run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql
 	fi
 	./init-db
@@ -206,7 +209,12 @@ main() {
 	import_schema --post-snapshot-import true --refresh-mviews true
 
 	step "Run snapshot validations."
-	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false'
+	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' || {
+			tail_log_file "yb-voyager-import-data.log"
+			tail_log_file "yb-voyager-export-data-from-source.log"
+			tail_log_file "yb-voyager-import-data-to-source-replica.log"
+			exit 1
+		}
 
 	step "Inserting new events to source"
 	run_sql_file source_delta.sql
@@ -286,7 +294,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }