Merge branch 'main' into shubham/unique-export-dir

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,37 @@
+### Describe the changes in this pull request
+
+
+### Describe if there are any user-facing changes
+<!--
+Clarify any changes to the user experience. For instance,
+  1. Were there any changes to the command line? 
+  2. Were there any changes to the configuration?
+  3. Has the installation process changed? 
+  4. Were there any changes to the reports? 
+-->
+
+### How was this pull request tested?
+<!--
+Mention if the existing tests were enough
+Mention the new unit tests added 
+Was any manual testing needed? If yes, is there a need to add integration tests for that?
+-->
+
+### Does your PR have changes that can cause upgrade issues? 
+| Component        | Breaking changes? |
+| :----------------------------------------------: | :-----------: |
+| MetaDB                                                    |  Yes/No |
+| Name registry json                                        |  Yes/No |
+| Data File Descriptor Json                                 |  Yes/No |
+| Export Snapshot Status Json                               |  Yes/No |
+| Import Data State                                         |  Yes/No |
+| Export Status Json                                        |  Yes/No |
+| Data .sql files of tables                                 |  Yes/No |
+| Export and import data queue                              |  Yes/No |
+| Schema Dump                                               |  Yes/No |
+| AssessmentDB                                              |  Yes/No |
+| Sizing DB                                                 |  Yes/No |
+| Migration Assessment Report Json                          |  Yes/No |
+| Callhome Json                                             |  Yes/No |
+| YugabyteD Tables                                          |  Yes/No |
+| TargetDB Metadata Tables                                  |  Yes/No |

.github/workflows/go.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Test
       run: |
         cd yb-voyager
-        go test -v ./...
+        go test -v  -skip 'TestDDLIssuesInYBVersion' ./... 
         
     - name: Vet
       run: |

.github/workflows/issue-tests.yml

@@ -0,0 +1,50 @@
+name: Go
+
+on:
+  push:
+    branches: ['main', '*.*-dev', '*.*.*-dev']
+  pull_request:
+    branches: [main]
+
+jobs:
+
+  test-issues-against-all-yb-versions:
+    strategy:
+      matrix:
+        version: [2.23.1.0-b220, 2024.1.3.1-b8, 2.20.8.0-b53, 2.18.9.0-b17]
+    env:
+      YB_VERSION: ${{ matrix.version }}
+      YB_CONN_STR: "postgres://yugabyte:[email protected]:5433/yugabyte"
+
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: "1.23.1"
+
+    - name: Setup YugabyteDB
+      run: |
+        # using s3 release instead of docker image to allow testing against un-released versions
+        wget https://s3.us-west-2.amazonaws.com/releases.yugabyte.com/${YB_VERSION}/yugabyte-${YB_VERSION}-centos-x86_64.tar.gz
+        mkdir -p yugabyte-${YB_VERSION}
+        tar -xvzf yugabyte-${YB_VERSION}-centos-x86_64.tar.gz -C yugabyte-${YB_VERSION} --strip-components=1
+        yugabyte-${YB_VERSION}/bin/yugabyted start --advertise_address 127.0.0.1
+        sleep 20
+    
+    - name: Test YugabyteDB connection
+      run: |
+        psql "${YB_CONN_STR}" -c "SELECT version();"
+
+    - name: Build
+      run: |
+        cd yb-voyager
+        go build -v ./...
+    
+    - name: Test Issues Against YB Version
+      run: |
+        cd yb-voyager
+        go test -v -run '^TestDDLIssuesInYBVersion$' ./...
+        

.github/workflows/misc-migtests.yml

@@ -71,11 +71,10 @@ jobs:
           ./migtests/scripts/postgresql/create_pg_user
 
       - name: "TEST: Assessment Report Test"
-        if: always()
         run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test
 
       - name: "TEST: analyze-schema"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/tests/analyze-schema/run-analyze-schema-test
 
       - name: Run import data file tests on different YugabyteDB versions
@@ -85,15 +84,17 @@ jobs:
           GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }}
           AWS_ACCESS_KEY_ID: ${{ secrets.RAHULB_S3_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }}
-        if: always()
+        if: ${{ !cancelled() }}
         run: |
           versions=("2.20.5.0-b72" "2.21.1.0-b271" "2024.1.1.0-b137")
           for version in "${versions[@]}"; do
             echo "Running tests on version $version"
 
             echo "Start YugabyteDB cluster"
-            docker pull yugabytedb/yugabyte:$version
-            VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+            docker run -d --name yugabytedb-$version \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:$version \
+            bin/yugabyted start --background=false --ui=false
             sleep 20
           
             echo "Test YugabyteDB connection"
@@ -114,8 +115,8 @@ jobs:
             migtests/tests/import-file/run-import-file-test
 
             echo "Stop the cluster before the next iteration"
-            VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml down --volumes
-            docker network prune -f
+            docker stop yugabytedb-$version
+            docker remove yugabytedb-$version
           done
         shell: bash
 

.github/workflows/mysql-migtests.yml

@@ -30,6 +30,7 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+
       - name: Install python3 and psycopg2
         run: |
           sudo apt install -y python3
@@ -59,8 +60,10 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker pull yugabytedb/yugabyte:${{ matrix.version }}
-          VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+          docker run -d --name yugabytedb \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:${{ matrix.version }} \
+            bin/yugabyted start --background=false --ui=false
           sleep 20
 
       - name: Test YugabyteDB connection
@@ -77,59 +80,59 @@ jobs:
           psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: mysql-table-list-flags-test (table-list and exclude-table-list)"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/table-list-flags-tests
 
       - name: "TEST: mysql-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/table-list-flags-tests env-file-path-flags.sh
 
       - name: "TEST: mysql-sakila"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/sakila
 
       - name: "TEST: mysql-datatypes"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/datatypes
 
       - name: "TEST: mysql-constraints"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/constraints
 
       - name: "TEST: mysql-case-indexes"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/indexes
 
       - name: "TEST: mysql-functions"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/functions
 
       - name: "TEST: mysql-case-sequences"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/sequences
 
       - name: "TEST: mysql-triggers-procedures"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/triggers-procedures
 
       - name: "TEST: mysql-case-views"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/views
 
       - name: "TEST: mysql-partitions"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/partitions
 
       - name: "TEST: mysql-sample-chinook"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/chinook
 
       - name: "TEST: mysql-misc-tests"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/misc-tests
 
       - name: "TEST: mysql-case-sensitivity-reserved-words"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/case-sensitivity-reserved-words
 
       # Placeholder for now so that a basic test can run
@@ -139,5 +142,5 @@ jobs:
           mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';'
 
       - name: "TEST: mysql-live-migration-test"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/live-migration-run-test.sh mysql/basic-live-test

.github/workflows/pg-9-migtests.yml

@@ -45,6 +45,7 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+
       - name: "Enable postgres with wal_level as logical"
         run: |
           docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf"
@@ -79,8 +80,10 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker pull yugabytedb/yugabyte:${{ matrix.version }}
-          VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+          docker run -d --name yugabytedb \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:${{ matrix.version }} \
+            bin/yugabyted start --background=false --ui=false
           sleep 20
 
       - name: Test YugabyteDB connection
@@ -98,16 +101,13 @@ jobs:
           psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: pg-case-sensitivity-single-table"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table
 
       - name: "TEST: pg-datatypes"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh pg/datatypes
 
       - name: "TEST: pg-constraints"
-        if: always()
-        run: migtests/scripts/run-test.sh pg/constraints
-
-
-
+        if: ${{ !cancelled() }}
+        run: migtests/scripts/run-test.sh pg/constraints