Github actions workflow optimisations (#1963)

* Divided pg tests in gh actions into 3 groups: offline, live_basic, live_advanced
- this will help to reduce the overall time taken for the GHA to complete
- Improvement: total time 57min to 26min i.e. 2x improvement

* Replaced always() with !cancelled() condition from if expressions of all tests

* Using yugabyted cmd to start ybdb cluster in docker container
- remove old way of creating using /yb-master and /yb-tserver commands

.github/workflows/misc-migtests.yml

@@ -71,11 +71,10 @@ jobs:
           ./migtests/scripts/postgresql/create_pg_user
 
       - name: "TEST: Assessment Report Test"
-        if: always()
         run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test
 
       - name: "TEST: analyze-schema"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/tests/analyze-schema/run-analyze-schema-test
 
       - name: Run import data file tests on different YugabyteDB versions
@@ -85,15 +84,17 @@ jobs:
           GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }}
           AWS_ACCESS_KEY_ID: ${{ secrets.RAHULB_S3_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }}
-        if: always()
+        if: ${{ !cancelled() }}
         run: |
           versions=("2.20.5.0-b72" "2.21.1.0-b271" "2024.1.1.0-b137")
           for version in "${versions[@]}"; do
             echo "Running tests on version $version"
 
             echo "Start YugabyteDB cluster"
-            docker pull yugabytedb/yugabyte:$version
-            VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+            docker run -d --name yugabytedb-$version \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:$version \
+            bin/yugabyted start --background=false --ui=false
             sleep 20
           
             echo "Test YugabyteDB connection"
@@ -114,8 +115,8 @@ jobs:
             migtests/tests/import-file/run-import-file-test
 
             echo "Stop the cluster before the next iteration"
-            VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml down --volumes
-            docker network prune -f
+            docker stop yugabytedb-$version
+            docker remove yugabytedb-$version
           done
         shell: bash
 

.github/workflows/mysql-migtests.yml

@@ -30,6 +30,7 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+
       - name: Install python3 and psycopg2
         run: |
           sudo apt install -y python3
@@ -59,8 +60,10 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker pull yugabytedb/yugabyte:${{ matrix.version }}
-          VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+          docker run -d --name yugabytedb \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:${{ matrix.version }} \
+            bin/yugabyted start --background=false --ui=false
           sleep 20
 
       - name: Test YugabyteDB connection
@@ -77,59 +80,59 @@ jobs:
           psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: mysql-table-list-flags-test (table-list and exclude-table-list)"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/table-list-flags-tests
 
       - name: "TEST: mysql-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/table-list-flags-tests env-file-path-flags.sh
 
       - name: "TEST: mysql-sakila"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/sakila
 
       - name: "TEST: mysql-datatypes"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/datatypes
 
       - name: "TEST: mysql-constraints"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/constraints
 
       - name: "TEST: mysql-case-indexes"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/indexes
 
       - name: "TEST: mysql-functions"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/functions
 
       - name: "TEST: mysql-case-sequences"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/sequences
 
       - name: "TEST: mysql-triggers-procedures"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/triggers-procedures
 
       - name: "TEST: mysql-case-views"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/views
 
       - name: "TEST: mysql-partitions"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/partitions
 
       - name: "TEST: mysql-sample-chinook"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/chinook
 
       - name: "TEST: mysql-misc-tests"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/misc-tests
 
       - name: "TEST: mysql-case-sensitivity-reserved-words"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh mysql/case-sensitivity-reserved-words
 
       # Placeholder for now so that a basic test can run
@@ -138,5 +141,5 @@ jobs:
           mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';'
 
       - name: "TEST: mysql-live-migration-test"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/live-migration-run-test.sh mysql/basic-live-test

.github/workflows/pg-9-migtests.yml

@@ -45,6 +45,7 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+
       - name: "Enable postgres with wal_level as logical"
         run: |
           docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf"
@@ -79,8 +80,10 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker pull yugabytedb/yugabyte:${{ matrix.version }}
-          VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+          docker run -d --name yugabytedb \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:${{ matrix.version }} \
+            bin/yugabyted start --background=false --ui=false
           sleep 20
 
       - name: Test YugabyteDB connection
@@ -98,16 +101,13 @@ jobs:
           psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: pg-case-sensitivity-single-table"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table
 
       - name: "TEST: pg-datatypes"
-        if: always()
+        if: ${{ !cancelled() }}
         run: migtests/scripts/run-test.sh pg/datatypes
 
       - name: "TEST: pg-constraints"
-        if: always()
-        run: migtests/scripts/run-test.sh pg/constraints
-
-
-
+        if: ${{ !cancelled() }}
+        run: migtests/scripts/run-test.sh pg/constraints

.github/workflows/pg-migtests.yml

@@ -12,6 +12,11 @@ jobs:
       matrix:
         version: [2.21.1.0-b271, 2024.1.1.0-b137, 2.20.5.0-b72]
         BETA_FAST_DATA_EXPORT: [0, 1]
+        test_group:
+          - offline
+          - live_basic
+          - live_advanced
+
     env:
       BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }}
     runs-on: ubuntu-22.04
@@ -38,19 +43,21 @@ jobs:
           distribution: "temurin"
           java-version: "17"
           check-latest: true
+
       - name: Cache local Maven repository
         uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+
       - name: "Enable postgres with wal_level as logical"
         run: |
           docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf"
           docker restart ${{ job.services.postgres.id }}
           sleep 10
-      #      if: matrix.BETA_FAST_DATA_EXPORT == 1
+
       - name: Install python3 and psycopg2
         run: |
           sudo apt install -y python3
@@ -79,8 +86,10 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker pull yugabytedb/yugabyte:${{ matrix.version }}
-          VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d
+          docker run -d --name yugabytedb \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:${{ matrix.version }} \
+            bin/yugabyted start --background=false --ui=false
           sleep 20
 
       - name: Test YugabyteDB connection
@@ -98,120 +107,112 @@ jobs:
           psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/table-list-flags-tests
 
       - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh
 
       - name: "TEST: pg-case-sensitivity-single-table"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table
 
       - name: "TEST: pg-dvdrental"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/dvdrental
 
       - name: "TEST: pg-datatypes"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/datatypes
 
       - name: "TEST: pg-constraints"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/constraints
 
       - name: "TEST: pg-sequences"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/sequences
 
       - name: "TEST: pg-indexes"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/indexes
 
       - name: "TEST: pg-partitions"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/partitions 
 
       - name: "TEST: pg-partitions with (table-list)"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions 
 
       # Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529
       # Fixed in 2.17.1.0-b368
       - name: "TEST: pg-partitions-with-indexes"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/partitions-with-indexes
 
       - name: "TEST: pg-views-and-rules"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/views-and-rules
 
       - name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/misc-objects-1
 
       - name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/misc-objects-2
 
       - name: "TEST: pg-dependent-ddls"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/dependent-ddls
 
       - name: "TEST: pg-multiple-schemas"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/multiple-schemas
 
-      - name: "Set up gcp environment"
-        env:
-          GCS_CLIENT_ID: ${{ secrets.PGUPTA_GCS_CLIENT_ID }}
-          GCS_CLIENT_SECRET: ${{ secrets.PGUPTA_GCS_CLIENT_SECRET }}
-          GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }}
-        if: always()
-        run: migtests/scripts/gcs/create_gcs_credentials_file
-
       - name: "TEST: pg-codependent-schemas"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/codependent-schemas
 
       - name: "TEST: pg-sample-schema-emp"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/sample-employee
 
       - name: "TEST: pg-hasura-ecommerce"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
         run: migtests/scripts/run-test.sh pg/hasura-ecommerce
 
+      - name: "TEST: pg-case-sensitivity-reserved-words-offline"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words
+
       - name: "TEST: pg-basic-non-public-live-migration-test"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
         run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test
-
-      - name: "TEST: pg-unique-key-conflicts-test"
-        if: always()
-        run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test
-
-      # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables.
-      - name: "TEST: pg-live-migration-multiple-schemas"
-        if: always()
-        run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh  pg/multiple-schemas
 
       - name: "TEST: pg-basic-public-fall-forward-test"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
         run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test
 
       # - name: "TEST: pg-basic-non-public-fall-back-test"
-      #   if: always()
       #   run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test
 
       - name: "TEST: pg-datatypes-fall-back-test"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
         run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes
 
+      # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables.
+      - name: "TEST: pg-live-migration-multiple-schemas"
+        if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
+        run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh  pg/multiple-schemas
+
+      - name: "TEST: pg-unique-key-conflicts-test"
+        if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
+        run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test
+
       - name: "TEST: pg-live-migration-partitions-fall-forward"
-        if: always()
+        if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
         run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions
 
-      - name: "TEST: pg-case-sensitivity-reserved-words-offline"
-        if: always()
-        run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words