Skip to content

Commit

Permalink
Merge branch 'main' into priyanshi/remove-guardrail-batch-size
Browse files Browse the repository at this point in the history
  • Loading branch information
priyanshi-yb committed Jan 15, 2025
2 parents 364d07e + 8b91860 commit db97752
Show file tree
Hide file tree
Showing 247 changed files with 23,290 additions and 5,723 deletions.
37 changes: 37 additions & 0 deletions .github/PULL_REQUEST_TEMPLATE
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
### Describe the changes in this pull request


### Describe if there are any user-facing changes
<!--
Clarify any changes to the user experience. For instance,
1. Were there any changes to the command line?
2. Were there any changes to the configuration?
3. Has the installation process changed?
4. Were there any changes to the reports?
-->

### How was this pull request tested?
<!--
Mention if the existing tests were enough
Mention the new unit tests added
Was any manual testing needed? If yes, is there a need to add integration tests for that?
-->

### Does your PR have changes that can cause upgrade issues?
| Component | Breaking changes? |
| :----------------------------------------------: | :-----------: |
| MetaDB | Yes/No |
| Name registry json | Yes/No |
| Data File Descriptor Json | Yes/No |
| Export Snapshot Status Json | Yes/No |
| Import Data State | Yes/No |
| Export Status Json | Yes/No |
| Data .sql files of tables | Yes/No |
| Export and import data queue | Yes/No |
| Schema Dump | Yes/No |
| AssessmentDB | Yes/No |
| Sizing DB | Yes/No |
| Migration Assessment Report Json | Yes/No |
| Callhome Json | Yes/No |
| YugabyteD Tables | Yes/No |
| TargetDB Metadata Tables | Yes/No |
4 changes: 2 additions & 2 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:

jobs:

build:
build-and-test:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
Expand All @@ -26,7 +26,7 @@ jobs:
- name: Test
run: |
cd yb-voyager
go test -v ./...
go test -v ./... -tags 'unit'
- name: Vet
run: |
Expand Down
53 changes: 53 additions & 0 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Go

on:
push:
branches: ['main', '*.*-dev', '*.*.*-dev']
pull_request:
branches: [main]

env:
ORACLE_INSTANT_CLIENT_VERSION: "21.5.0.0.0-1"

jobs:
integration-tests:
strategy:
fail-fast: false

runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3

- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: "1.23.1"

- name: Build
run: |
cd yb-voyager
go build -v ./...
# required by godror driver used in the tests
- name: Install Oracle Instant Clients
run: |
# Download and install the YB APT repository package
wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/reporpms/yb-apt-repo_1.0.0_all.deb
sudo apt-get install -y ./yb-apt-repo_1.0.0_all.deb
sudo apt-get update -y
# Install Oracle Instant Client packages using the defined version
sudo apt-get install -y oracle-instantclient-tools=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
sudo apt-get install -y oracle-instantclient-basic=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
sudo apt-get install -y oracle-instantclient-devel=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
sudo apt-get install -y oracle-instantclient-jdbc=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
sudo apt-get install -y oracle-instantclient-sqlplus=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
# Clean up the YB APT repository package
sudo apt-get remove -y yb-apt-repo
rm -f yb-apt-repo_1.0.0_all.deb
- name: Run Integration Tests
run: |
cd yb-voyager
go test -v ./... -tags 'integration'
50 changes: 50 additions & 0 deletions .github/workflows/issue-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: Go

on:
push:
branches: ['main', '*.*-dev', '*.*.*-dev']
pull_request:
branches: [main]

jobs:

test-issues-against-all-yb-versions:
strategy:
fail-fast: false
matrix:
version: [2.23.1.0-b220, 2024.1.3.1-b8, 2024.2.0.0-b145, 2.20.8.0-b53, 2.18.9.0-b17]
env:
YB_VERSION: ${{ matrix.version }}
YB_CONN_STR: "postgres://yugabyte:[email protected]:5433/yugabyte"

runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3

- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: "1.23.1"

- name: Setup YugabyteDB
run: |
# using s3 release instead of docker image to allow testing against un-released versions
wget https://s3.us-west-2.amazonaws.com/releases.yugabyte.com/${YB_VERSION}/yugabyte-${YB_VERSION}-centos-x86_64.tar.gz
mkdir -p yugabyte-${YB_VERSION}
tar -xvzf yugabyte-${YB_VERSION}-centos-x86_64.tar.gz -C yugabyte-${YB_VERSION} --strip-components=1
yugabyte-${YB_VERSION}/bin/yugabyted start --advertise_address 127.0.0.1
sleep 20
- name: Test YugabyteDB connection
run: |
psql "${YB_CONN_STR}" -c "SELECT version();"
- name: Build
run: |
cd yb-voyager
go build -v ./...
- name: Test Issues Against YB Version
run: |
cd yb-voyager
go test -v ./... -tags 'issues_integration'
29 changes: 18 additions & 11 deletions .github/workflows/misc-migtests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ jobs:

services:
postgres:
image: postgres:13
image: postgres:17
env:
POSTGRES_PASSWORD: secret
POSTGRES_PASSWORD: postgres
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
Expand Down Expand Up @@ -55,27 +55,32 @@ jobs:
sudo apt install -y libpq-dev
sudo apt install python3-psycopg2
#TODO Remove the install PG 17 command once we do that in installer script
- name: Run installer script to setup voyager
run: |
cd installer_scripts
yes | ./install-yb-voyager --install-from-local-source --only-pg-support
sudo apt-get -y install postgresql-17
echo "/usr/lib/postgresql/17/bin" >> "$GITHUB_PATH"
env:
ON_INSTALLER_ERROR_OUTPUT_LOG: Y

- name: Test PostgreSQL Connection
run: |
psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();"
psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();"
- name: Create PostgreSQL user
run: |
./migtests/scripts/postgresql/create_pg_user
- name: "TEST: Assessment Report Test"
if: always()
run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test

- name: "TEST: Assessment Report Test (Schema list UQC)"
run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test-uqc

- name: "TEST: analyze-schema"
if: always()
if: ${{ !cancelled() }}
run: migtests/tests/analyze-schema/run-analyze-schema-test

- name: Run import data file tests on different YugabyteDB versions
Expand All @@ -85,15 +90,17 @@ jobs:
GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.RAHULB_S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }}
if: always()
if: ${{ !cancelled() }}
run: |
versions=("2.20.5.0-b72" "2.21.1.0-b271" "2024.1.1.0-b137")
versions=("2024.2.0.0-b145" "2.20.8.0-b53" "2024.1.3.1-b8" "2.23.1.0-b220")
for version in "${versions[@]}"; do
echo "Running tests on version $version"
echo "Start YugabyteDB cluster"
docker pull yugabytedb/yugabyte:$version
VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml up -d
docker run -d --name yugabytedb-$version \
-p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
yugabytedb/yugabyte:$version \
bin/yugabyted start --background=false --ui=false
sleep 20
echo "Test YugabyteDB connection"
Expand All @@ -114,8 +121,8 @@ jobs:
migtests/tests/import-file/run-import-file-test
echo "Stop the cluster before the next iteration"
VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml down --volumes
docker network prune -f
docker stop yugabytedb-$version
docker remove yugabytedb-$version
done
shell: bash

40 changes: 22 additions & 18 deletions .github/workflows/mysql-migtests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
run-mysql-migration-tests:
strategy:
matrix:
version: [2.21.1.0-b271, 2024.1.1.0-b137, 2.20.5.0-b72]
version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
BETA_FAST_DATA_EXPORT: [0, 1]
env:
BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }}
Expand All @@ -30,6 +30,7 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Install python3 and psycopg2
run: |
sudo apt install -y python3
Expand Down Expand Up @@ -59,8 +60,10 @@ jobs:
- name: Start YugabyteDB cluster
run: |
docker pull yugabytedb/yugabyte:${{ matrix.version }}
VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d
docker run -d --name yugabytedb \
-p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
yugabytedb/yugabyte:${{ matrix.version }} \
bin/yugabyted start --background=false --ui=false
sleep 20
- name: Test YugabyteDB connection
Expand All @@ -77,66 +80,67 @@ jobs:
psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
- name: "TEST: mysql-table-list-flags-test (table-list and exclude-table-list)"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/table-list-flags-tests

- name: "TEST: mysql-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/table-list-flags-tests env-file-path-flags.sh

- name: "TEST: mysql-sakila"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/sakila

- name: "TEST: mysql-datatypes"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/datatypes

- name: "TEST: mysql-constraints"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/constraints

- name: "TEST: mysql-case-indexes"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/indexes

- name: "TEST: mysql-functions"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/functions

- name: "TEST: mysql-case-sequences"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/sequences

- name: "TEST: mysql-triggers-procedures"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/triggers-procedures

- name: "TEST: mysql-case-views"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/views

- name: "TEST: mysql-partitions"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/partitions

- name: "TEST: mysql-sample-chinook"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/chinook

- name: "TEST: mysql-misc-tests"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/misc-tests

- name: "TEST: mysql-case-sensitivity-reserved-words"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/run-test.sh mysql/case-sensitivity-reserved-words

# Placeholder for now so that a basic test can run
- name: Create the live migration user
if: always()
run: |
mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';'
- name: "TEST: mysql-live-migration-test"
if: always()
if: ${{ !cancelled() }}
run: migtests/scripts/live-migration-run-test.sh mysql/basic-live-test
Loading

0 comments on commit db97752

Please sign in to comment.