From bcb00466c53a04d5c3169a0dfb038ad8e8c61d49 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 12 Feb 2024 15:29:08 +0100 Subject: [PATCH] introduce --test-suite to better control what tests need to run --- .github/workflows/ci.yml | 4 ++-- .github/workflows/e2e.yml | 10 ++++++--- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- build/scripts/CommandLine.fs | 30 +++++++++++++------------- build/scripts/Targets.fs | 37 +++++++++++++++++--------------- 6 files changed, 46 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d877e51..32513c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: id: bootstrap uses: ./.github/workflows/bootstrap - - run: build.bat test + - run: build.bat test --test-suite=skip-e2e shell: cmd name: Test @@ -46,5 +46,5 @@ jobs: uses: ./.github/workflows/bootstrap # We still run the full release build on pull-requests this ensures packages are validated ahead of time - - run: ./build.sh release + - run: ./build.sh release --test-suite=skip-e2e name: Release diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c1ee6f6..c15a26d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,6 +51,10 @@ jobs: vault-role-id: ${{ secrets.VAULT_ROLE_ID }} vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} - # TODO: run the e2e targeting the required endpoint. - # those values can be found in https://github.com/elastic/apm-pipeline-library/tree/main/.github/actions/oblt-cli-cluster-credentials#outputs - - run: curl -X GET "${ELASTICSEARCH_HOST}/_cat/indices?v" -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} + + - run: ./build.sh test --test-suite=e2e + env: + E2E__ENDPOINT: "${ELASTIC_APM_SERVER_URL}" + E2E__Authorization: "Authentication=ApiKey ${ELASTIC_APM_API_KEY}" + E2E__BROWSEREMAIL: "${KIBANA_USERNAME}" + E2E__BROWSERPASSWORD: "${KIBANA_PASSWORD}" diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index f73cc0e..5211a00 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -21,7 +21,7 @@ jobs: id: bootstrap uses: ./.github/workflows/bootstrap - - run: ./build.sh release + - run: ./build.sh release --test-suite=skip-e2e name: Release - name: publish canary packages github package repository diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc6db8b..a81be4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: id: bootstrap uses: ./.github/workflows/bootstrap - - run: ./build.sh release --skiptests + - run: ./build.sh release --test-suite=skip-all name: Release - name: Prepare Nuget diff --git a/build/scripts/CommandLine.fs b/build/scripts/CommandLine.fs index 944bbef..27133b0 100644 --- a/build/scripts/CommandLine.fs +++ b/build/scripts/CommandLine.fs @@ -9,15 +9,16 @@ open Microsoft.FSharp.Reflection open System open Bullseye +type TestSuite = All | Unit | Integration | E2E | Skip_All | Skip_E2E + type Build = | [] Clean | [] Version | [] Build | [] Test - | [] UnitTest - | [] Integrate - | [] EndToEnd + | [] Unit_Test + | [] End_To_End | [] PristineCheck | [] GeneratePackages @@ -27,10 +28,10 @@ type Build = | [] GenerateApiChanges | [] Release - | [] SingleTarget + | [] Single_Target | [] Token of string - | [] SkipDirtyCheck - | [] SkipTests + | [] Skip_Dirty_Check + | [] Test_Suite of TestSuite with interface IArgParserTemplate with member this.Usage = @@ -40,11 +41,9 @@ with | Version -> "print version information" | Build -> "Run build" - | UnitTest -> "build then run only the unit tests" - | Integrate -> "build then run only the integration tests" - | EndToEnd -> "build then run only the e2e tests" - | Test -> "build then run ALL the tests" - + | Unit_Test -> "alias to providing: test --test-suite=unit" + | End_To_End -> "alias to providing: test --test-suite=e2e" + | Test -> "runs a clean build and then runs all the tests unless --test-suite is provided" | Release -> "runs build, tests, and create and validates the packages shy of publishing them" // steps @@ -56,10 +55,11 @@ with | GenerateApiChanges -> "Undocumented, dependent target" // flags - | SingleTarget -> "Runs the provided sub command without running their dependencies" + | Single_Target -> "Runs the provided sub command without running their dependencies" | Token _ -> "Token to be used to authenticate with github" - | SkipDirtyCheck -> "Skip the clean checkout check that guards the release/publish targets" - | SkipTests -> "Skips running tests" + | Skip_Dirty_Check -> "Skip the clean checkout check that guards the release/publish targets" + | Test_Suite _ -> "Specify the test suite to run, defaults to all" + member this.StepName = match FSharpValue.GetUnionFields(this, typeof) with @@ -79,7 +79,7 @@ with Targets.Target(target.StepName, Action(fun _ -> action(parsed))) static member Cmd (dependsOn: Build list) (composedOf: Build list) action (target: Build) (parsed: ParseResults) = - let singleTarget = parsed.TryGetResult SingleTarget |> Option.isSome + let singleTarget = parsed.TryGetResult Single_Target |> Option.isSome let dependsOn = if singleTarget then [] else dependsOn let steps = dependsOn @ composedOf |> List.map (_.StepName) diff --git a/build/scripts/Targets.fs b/build/scripts/Targets.fs index 5d8e29a..b656435 100644 --- a/build/scripts/Targets.fs +++ b/build/scripts/Targets.fs @@ -34,14 +34,12 @@ let private version _ = let private generatePackages _ = exec { run "dotnet" "pack" } let private pristineCheck (arguments:ParseResults) = - let skipCheck = arguments.TryGetResult SkipDirtyCheck |> Option.isSome + let skipCheck = arguments.TryGetResult Skip_Dirty_Check |> Option.isSome match skipCheck, Information.isCleanWorkingCopy "." with | true, _ -> printfn "Checkout is dirty but -c was specified to ignore this" | _, true -> printfn "The checkout folder does not have pending changes, proceeding" | _ -> failwithf "The checkout folder has pending changes, aborting. Specify -c to ./build.sh to skip this check" -type TestSuite = | Unit | Integration | E2E | All - let private runTests suite _ = let logger = // use junit xml logging locally, github actions logs using console out formats @@ -55,9 +53,11 @@ let private runTests suite _ = let filterArgs = match suite with | All -> [] - | TestSuite.Unit -> [ "--filter"; "FullyQualifiedName~.Tests" ] - | TestSuite.Integration -> [ "--filter"; "FullyQualifiedName~.IntegrationTests" ] - | TestSuite.E2E -> [ "--filter"; "FullyQualifiedName~.EndToEndTests" ] + | Skip_All -> ["--filter"; "FullyQualifiedName~.SKIPPING.ALL.TESTS"] + | Unit -> [ "--filter"; "FullyQualifiedName~.Tests" ] + | Integration -> [ "--filter"; "FullyQualifiedName~.IntegrationTests" ] + | E2E -> [ "--filter"; "FullyQualifiedName~.EndToEndTests" ] + | Skip_E2E -> [ "--filter"; "FullyQualifiedName!~.EndToEndTests" ] let settingsArg = ["-s"; "tests/.runsettings"] @@ -72,10 +72,14 @@ let private runTests suite _ = ) } -let private test suite (arguments:ParseResults) = - match arguments.TryGetResult SkipTests with - | None -> runTests suite arguments - | Some _ -> printfn "Skipping tests because --skiptests was provided" +let private test (arguments:ParseResults) = + let arg = arguments.TryGetResult Test_Suite + match arg with + | None -> runTests TestSuite.All arguments + | Some suite -> + match suite with + | Skip_All -> printfn "Skipping tests because --test-suite skip was provided" + | _ -> runTests suite arguments let private validateLicenses _ = let args = ["-u"; "-t"; "-i"; "Elastic.OpenTelemetry.sln"; "--use-project-assets-json" @@ -149,10 +153,9 @@ let Setup (parsed:ParseResults) = | Clean -> Build.Cmd [Version] [] clean | Build -> Build.Cmd [Clean] [] build - | UnitTest -> Build.Cmd [Build] [] <| test TestSuite.Unit - | Integrate -> Build.Cmd [Build] [] <| test TestSuite.Integration - | EndToEnd -> Build.Cmd [Build] [] <| test TestSuite.E2E - | Test -> Build.Cmd [UnitTest; Integrate; EndToEnd] [] ignore + | End_To_End -> Build.Cmd [] [Build] <| runTests E2E + | Unit_Test -> Build.Cmd [] [Build] <| runTests Unit + | Test -> Build.Cmd [] [Build] test | Release -> Build.Cmd @@ -169,10 +172,10 @@ let Setup (parsed:ParseResults) = | GenerateApiChanges -> Build.Step generateApiChanges // flags - | SingleTarget - | SkipTests + | Single_Target + | Test_Suite _ | Token _ - | SkipDirtyCheck -> Build.Ignore + | Skip_Dirty_Check -> Build.Ignore for target in Build.Targets do let setup = wireCommandLine target