fix(tests): run tests like in CI, make tests pass locally

.config/nextest.toml

@@ -12,6 +12,30 @@ retries = 1
 # Timeout tests after 5 minutes
 slow-timeout = { period = "60s", terminate-after = 5 }
 
+# isolate slow and flaky tests to run alone (no multi-thread / no other tests running)
+[test-groups]
+no-concurrency = { max-threads = 1 }
+[[profile.ci.overrides]]
+filter = 'test(test_withdraw_stake) or test(verify_latest_tx_replay_testnet_mainnet)'
+test-group = 'no-concurrency'
+retries = 3
+slow-timeout = { period = "120s", terminate-after = 5 }
+[[profile.ci.overrides]]
+filter = 'test(generate_cursor_test)' # slow test
+test-group = 'no-concurrency'
+retries = 3
+slow-timeout = { period = "120s", terminate-after = 5 }
+[[profile.ci.overrides]]
+filter = 'test(test_reconfig_with_committee_change_stress_determinism)' # slow test
+test-group = 'no-concurrency'
+retries = 3
+slow-timeout = { period = "120s", terminate-after = 5 }
+[[profile.ci.overrides]]
+filter = 'package(iota-graphql-rpc) and test(test_epoch_data)' # flaky test
+test-group = 'no-concurrency'
+retries = 3
+slow-timeout = { period = "120s", terminate-after = 5 }
+
 [profile.simtestnightly]
 # Print out output for failing tests as soon as they fail, and also at the end
 # of the run (for easy scrollability).

.github/workflows/_rust_tests.yml

@@ -43,6 +43,7 @@ jobs:
       - uses: taiki-e/install-action@375e0c7f08a66b8c2ba7e7eef31a6f91043a81b0 # v2.44.38
         with:
           tool: nextest
+        # test using filters, to only include changed crates and all their dependent crates
       - name: cargo test
         run: |
           array=(${{ inputs.changedCrates }})

.gitignore

@@ -72,6 +72,7 @@ yarn-error.log*
 checkpoints_dir/*
 light_client.yaml
 scripts/dependency_graphs/output/*
+**/.cache/
 
 lcov.info
 **/venv/

scripts/tests_like_ci/rust_tests.sh

@@ -0,0 +1,98 @@
+# INPUTS
+WD=$(git rev-parse --show-toplevel)
+
+# restart postgres
+function restart_postgres() {
+    if ! command -v psql &> /dev/null; then
+        echo "'psql' is not installed in PATH. Please ensure it is installed and available."
+        exit 1
+    fi
+    # docker rm -f -v $(docker ps -a | grep postgres | awk '{print $1}')
+    # DOCKER_CMD=$(docker ps -a -q | xargs docker inspect --format='{{.Config.Env}}' | grep [i]ota_indexer)
+    # docker ps -a -q | xargs docker inspect --format='{{.ID}} {{.Config.Env}}' | grep -oP '(?<=\s)[0-9a-f]{12}(?=.*\[i]ota_indexer)' | uniq
+    # DOCKER_CMD=$(docker ps -a -q | xargs docker inspect --format='{{.Config}}' | grep -oP '(?<=com.docker.compose.project.config_files=).*docker-compose.yaml' | grep -q /root/src/iota/docker/pg-services-local/docker-compose.yaml && echo "true" || echo "false")
+    export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgrespw}
+    export POSTGRES_USER=${POSTGRES_USER:-postgres}
+    export POSTGRES_DB=${POSTGRES_DB:-iota_indexer}
+    export POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+    # assuming you run the indexer's postgres using docker-compose
+    cd ${WD}/docker/pg-services-local; docker-compose down -v postgres; docker-compose up -d postgres
+    PGPASSWORD=$POSTGRES_PASSWORD psql -h localhost -U $POSTGRES_USER -c 'CREATE DATABASE IF NOT EXISTS iota_indexer;' -c 'ALTER SYSTEM SET max_connections = 500;' 2>/dev/null
+}
+
+function retry_failing_only() {
+    filterset=""
+    for line in "${FAILING_NONSIM_TESTS[@]}"; do
+        arr=(${line// / })
+        if [ ${#arr[@]} -eq 2 ]; then
+            package=${arr[0]%%::*}
+            test_name=${arr[-1]#*::}
+            echo "package:$package test_name:$test_name"
+            # cargo nextest run --profile ci -E "package(${package}) and test(${test_name})"
+            # filterset="${filterset} -E 'package(${package}) and test(${test_name})'"
+            filterset="${filterset} -E 'test(${test_name})'"
+            break   
+        fi
+    done
+    echo "FILTERSET: ${filterset}"
+    command="cargo nextest run --profile ci ${filterset} --test-threads 1"
+    set -x
+    eval $command
+}
+
+# test rust crates
+function test_rust_crates() {
+    # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var
+    # causes #[sim_test] to only run under the deterministic `simtest` job, and not the
+    # non-deterministic `test` job.
+    export IOTA_SKIP_SIMTESTS=1
+    cargo nextest run --config-file .config/nextest.toml --profile ci
+}
+
+function test_external_crates() {
+    # rust / external-tests / Test external crates  https://github.com/iotaledger/iota/actions/runs/12752200362/job/35542165456
+    cargo nextest run  --config-file .config/nextest.toml   --manifest-path external-crates/move/Cargo.toml   -E '!test(prove) and !test(run_all::simple_build_with_docs/args.txt) and !test(run_test::nested_deps_bad_parent/Move.toml)'      --profile ci
+}
+
+
+# TODO check udeps
+
+# test-extra
+function test_extra() {
+    export IOTA_SKIP_SIMTESTS=1
+    cargo run --package iota-benchmark --bin stress -- --log-path ${WD}/.cache/stress.log --num-client-threads 10 --num-server-threads 24 --num-transfer-accounts 2 bench --target-qps 100 --num-workers 10  --transfer-object 50 --shared-counter 50 --run-duration 10s --stress-stat-collection
+    cargo test --doc
+    cargo doc --all-features --workspace --no-deps
+    ${WD}/scripts/execution_layer.py generate-lib;
+    ${WD}/scripts/changed-files.sh;
+}
+
+# simtest
+function simtests() {
+    # set -x
+    export MSIM_WATCHDOG_TIMEOUT_MS=60000
+    # TODO in 4665 use cargo-hakari
+    scripts/simtest/cargo-simtest simtest --profile ci --color always
+    scripts/simtest/stress-new-tests.sh
+}
+
+# tests using postgres
+function tests_using_postgres() {
+    restart_postgres
+    cargo nextest run --no-fail-fast --test-threads 1 --package iota-graphql-rpc --test e2e_tests --test examples_validation_tests --features pg_integration
+    cargo nextest run --no-fail-fast --test-threads 1 --package iota-graphql-rpc --lib --features pg_integration -- test_query_cost
+    cargo nextest run --no-fail-fast --test-threads 8 --package iota-graphql-e2e-tests --features pg_integration
+    cargo nextest run --no-fail-fast --test-threads 1 --package iota-cluster-test --test local_cluster_test --features pg_integration
+    cargo nextest run --no-fail-fast --test-threads 1 --package iota-indexer --test ingestion_tests --features pg_integration
+    # Iota-indexer's RPC tests, which depend on a shared runtime, are incompatible with nextest due to its process-per-test execution model.
+    # cargo test, on the other hand, allows tests to share state and resources by default.
+    cargo test --profile simulator --package iota-indexer --test rpc-tests --features shared_test_runtime
+}
+
+# run all steps
+set -euxo pipefail
+test_rust_crates
+test_external_crates
+test_extra
+tests_using_postgres
+simtests