From 2127d22599890f6dcb90eb22ef6402ab51424351 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Sat, 30 Nov 2024 11:03:10 +0530 Subject: [PATCH] Github actions workflow optimisations (#1963) * Divided pg tests in gh actions into 3 groups: offline, live_basic, live_advanced - this will help to reduce the overall time taken for the GHA to complete - Improvement: total time 57min to 26min i.e. 2x improvement * Replaced always() with !cancelled() condition from if expressions of all tests * Using yugabyted cmd to start ybdb cluster in docker container - remove old way of creating using /yb-master and /yb-tserver commands --- .github/workflows/misc-migtests.yml | 15 +++-- .github/workflows/mysql-migtests.yml | 37 ++++++----- .github/workflows/pg-9-migtests.yml | 18 ++--- .github/workflows/pg-migtests.yml | 95 ++++++++++++++------------- migtests/setup/yb-docker-compose.yaml | 46 ------------- 5 files changed, 85 insertions(+), 126 deletions(-) delete mode 100644 migtests/setup/yb-docker-compose.yaml diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 4a36f89aef..c56e4a00d2 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -71,11 +71,10 @@ jobs: ./migtests/scripts/postgresql/create_pg_user - name: "TEST: Assessment Report Test" - if: always() run: migtests/scripts/run-validate-assessment-report.sh pg/assessment-report-test - name: "TEST: analyze-schema" - if: always() + if: ${{ !cancelled() }} run: migtests/tests/analyze-schema/run-analyze-schema-test - name: Run import data file tests on different YugabyteDB versions @@ -85,15 +84,17 @@ jobs: GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.RAHULB_S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }} - if: always() + if: ${{ !cancelled() }} run: | versions=("2.20.5.0-b72" "2.21.1.0-b271" "2024.1.1.0-b137") for version in "${versions[@]}"; do echo "Running tests on version $version" echo "Start YugabyteDB cluster" - docker pull yugabytedb/yugabyte:$version - VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb-$version \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:$version \ + bin/yugabyted start --background=false --ui=false sleep 20 echo "Test YugabyteDB connection" @@ -114,8 +115,8 @@ jobs: migtests/tests/import-file/run-import-file-test echo "Stop the cluster before the next iteration" - VERSION=$version docker compose -f migtests/setup/yb-docker-compose.yaml down --volumes - docker network prune -f + docker stop yugabytedb-$version + docker remove yugabytedb-$version done shell: bash diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 7872bea0f5..d6b434a57b 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -30,6 +30,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: Install python3 and psycopg2 run: | sudo apt install -y python3 @@ -59,8 +60,10 @@ jobs: - name: Start YugabyteDB cluster run: | - docker pull yugabytedb/yugabyte:${{ matrix.version }} - VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false sleep 20 - name: Test YugabyteDB connection @@ -77,59 +80,59 @@ jobs: psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - name: "TEST: mysql-table-list-flags-test (table-list and exclude-table-list)" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/table-list-flags-tests - name: "TEST: mysql-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/table-list-flags-tests env-file-path-flags.sh - name: "TEST: mysql-sakila" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/sakila - name: "TEST: mysql-datatypes" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/datatypes - name: "TEST: mysql-constraints" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/constraints - name: "TEST: mysql-case-indexes" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/indexes - name: "TEST: mysql-functions" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/functions - name: "TEST: mysql-case-sequences" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/sequences - name: "TEST: mysql-triggers-procedures" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/triggers-procedures - name: "TEST: mysql-case-views" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/views - name: "TEST: mysql-partitions" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/partitions - name: "TEST: mysql-sample-chinook" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/chinook - name: "TEST: mysql-misc-tests" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/misc-tests - name: "TEST: mysql-case-sensitivity-reserved-words" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh mysql/case-sensitivity-reserved-words # Placeholder for now so that a basic test can run @@ -138,5 +141,5 @@ jobs: mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';' - name: "TEST: mysql-live-migration-test" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/live-migration-run-test.sh mysql/basic-live-test diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index 622ba4134a..458804c934 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -45,6 +45,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: "Enable postgres with wal_level as logical" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" @@ -79,8 +80,10 @@ jobs: - name: Start YugabyteDB cluster run: | - docker pull yugabytedb/yugabyte:${{ matrix.version }} - VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false sleep 20 - name: Test YugabyteDB connection @@ -98,16 +101,13 @@ jobs: psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - name: "TEST: pg-case-sensitivity-single-table" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - name: "TEST: pg-datatypes" - if: always() + if: ${{ !cancelled() }} run: migtests/scripts/run-test.sh pg/datatypes - name: "TEST: pg-constraints" - if: always() - run: migtests/scripts/run-test.sh pg/constraints - - - + if: ${{ !cancelled() }} + run: migtests/scripts/run-test.sh pg/constraints \ No newline at end of file diff --git a/.github/workflows/pg-migtests.yml b/.github/workflows/pg-migtests.yml index 6f5af1d821..cb2457f128 100644 --- a/.github/workflows/pg-migtests.yml +++ b/.github/workflows/pg-migtests.yml @@ -12,6 +12,11 @@ jobs: matrix: version: [2.21.1.0-b271, 2024.1.1.0-b137, 2.20.5.0-b72] BETA_FAST_DATA_EXPORT: [0, 1] + test_group: + - offline + - live_basic + - live_advanced + env: BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} runs-on: ubuntu-22.04 @@ -38,6 +43,7 @@ jobs: distribution: "temurin" java-version: "17" check-latest: true + - name: Cache local Maven repository uses: actions/cache@v3 with: @@ -45,12 +51,13 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: "Enable postgres with wal_level as logical" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" docker restart ${{ job.services.postgres.id }} sleep 10 - # if: matrix.BETA_FAST_DATA_EXPORT == 1 + - name: Install python3 and psycopg2 run: | sudo apt install -y python3 @@ -79,8 +86,10 @@ jobs: - name: Start YugabyteDB cluster run: | - docker pull yugabytedb/yugabyte:${{ matrix.version }} - VERSION=${{ matrix.version }} docker compose -f migtests/setup/yb-docker-compose.yaml up -d + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false sleep 20 - name: Test YugabyteDB connection @@ -98,120 +107,112 @@ jobs: psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/table-list-flags-tests - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh - name: "TEST: pg-case-sensitivity-single-table" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - name: "TEST: pg-dvdrental" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/dvdrental - name: "TEST: pg-datatypes" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/datatypes - name: "TEST: pg-constraints" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/constraints - name: "TEST: pg-sequences" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/sequences - name: "TEST: pg-indexes" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/indexes - name: "TEST: pg-partitions" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/partitions - name: "TEST: pg-partitions with (table-list)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions # Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529 # Fixed in 2.17.1.0-b368 - name: "TEST: pg-partitions-with-indexes" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/partitions-with-indexes - name: "TEST: pg-views-and-rules" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/views-and-rules - name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/misc-objects-1 - name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/misc-objects-2 - name: "TEST: pg-dependent-ddls" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/dependent-ddls - name: "TEST: pg-multiple-schemas" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/multiple-schemas - - name: "Set up gcp environment" - env: - GCS_CLIENT_ID: ${{ secrets.PGUPTA_GCS_CLIENT_ID }} - GCS_CLIENT_SECRET: ${{ secrets.PGUPTA_GCS_CLIENT_SECRET }} - GCS_REFRESH_TOKEN: ${{ secrets.PGUPTA_GCS_REFRESH_TOKEN }} - if: always() - run: migtests/scripts/gcs/create_gcs_credentials_file - - name: "TEST: pg-codependent-schemas" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/codependent-schemas - name: "TEST: pg-sample-schema-emp" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/sample-employee - name: "TEST: pg-hasura-ecommerce" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/hasura-ecommerce + - name: "TEST: pg-case-sensitivity-reserved-words-offline" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words + - name: "TEST: pg-basic-non-public-live-migration-test" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test - - - name: "TEST: pg-unique-key-conflicts-test" - if: always() - run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test - - # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables. - - name: "TEST: pg-live-migration-multiple-schemas" - if: always() - run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas - name: "TEST: pg-basic-public-fall-forward-test" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test # - name: "TEST: pg-basic-non-public-fall-back-test" - # if: always() # run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test - name: "TEST: pg-datatypes-fall-back-test" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes + # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables. + - name: "TEST: pg-live-migration-multiple-schemas" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas + + - name: "TEST: pg-unique-key-conflicts-test" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test + - name: "TEST: pg-live-migration-partitions-fall-forward" - if: always() + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions - - name: "TEST: pg-case-sensitivity-reserved-words-offline" - if: always() - run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words diff --git a/migtests/setup/yb-docker-compose.yaml b/migtests/setup/yb-docker-compose.yaml deleted file mode 100644 index e8c43262cd..0000000000 --- a/migtests/setup/yb-docker-compose.yaml +++ /dev/null @@ -1,46 +0,0 @@ -version: '2.1' - -volumes: - yb-master-data-1: - yb-tserver-data-1: - -services: - yb-master: - image: yugabytedb/yugabyte:${VERSION} - container_name: yb-master-n1 - volumes: - - yb-master-data-1:/mnt/master - command: [ "/home/yugabyte/bin/yb-master", - "--fs_data_dirs=/mnt/master", - "--master_addresses=yb-master-n1:7100", - "--rpc_bind_addresses=yb-master-n1:7100", - "--replication_factor=1"] - ports: - - "7000:7000" - - "7100:7100" - environment: - SERVICE_7000_NAME: yb-master - - yb-tserver: - image: yugabytedb/yugabyte:${VERSION} - container_name: yb-tserver-n1 - volumes: - - yb-tserver-data-1:/mnt/tserver - command: [ "/home/yugabyte/bin/yb-tserver", - "--fs_data_dirs=/mnt/tserver", - "--start_pgsql_proxy", - "--rpc_bind_addresses=yb-tserver-n1:9100", - "--tserver_master_addrs=yb-master-n1:7100"] - ports: - - "9042:9042" - - "5433:5433" - - "9000:9000" - - "9100:9100" - environment: - SERVICE_5433_NAME: ysql - SERVICE_9042_NAME: ycql - SERVICE_6379_NAME: yedis - SERVICE_9000_NAME: yb-tserver - depends_on: - - yb-master -