introduce --test-suite to better control what tests need to run

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
         id: bootstrap
         uses: ./.github/workflows/bootstrap
 
-      - run: build.bat test
+      - run: build.bat test --test-suite=skip-e2e
         shell: cmd
         name: Test
 
@@ -46,5 +46,5 @@ jobs:
         uses: ./.github/workflows/bootstrap
 
       # We still run the full release build on pull-requests this ensures packages are validated ahead of time
-      - run: ./build.sh release
+      - run: ./build.sh release --test-suite=skip-e2e
         name: Release

.github/workflows/e2e.yml

@@ -51,6 +51,10 @@ jobs:
           vault-role-id: ${{ secrets.VAULT_ROLE_ID }}
           vault-secret-id: ${{ secrets.VAULT_SECRET_ID }}
 
-      # TODO: run the e2e targeting the required endpoint.
-      #       those values can be found in https://github.com/elastic/apm-pipeline-library/tree/main/.github/actions/oblt-cli-cluster-credentials#outputs
-      - run: curl -X GET "${ELASTICSEARCH_HOST}/_cat/indices?v" -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}
+
+      - run: ./build.sh test --test-suite=e2e
+        env: 
+          E2E__ENDPOINT: "${ELASTIC_APM_SERVER_URL}"
+          E2E__Authorization: "Authentication=ApiKey ${ELASTIC_APM_API_KEY}"
+          E2E__BROWSEREMAIL: "${KIBANA_USERNAME}"
+          E2E__BROWSERPASSWORD: "${KIBANA_PASSWORD}"

.github/workflows/prerelease.yml

@@ -21,7 +21,7 @@ jobs:
         id: bootstrap
         uses: ./.github/workflows/bootstrap
 
-      - run: ./build.sh release
+      - run: ./build.sh release --test-suite=skip-e2e
         name: Release
 
       - name: publish canary packages github package repository

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
       id: bootstrap
       uses: ./.github/workflows/bootstrap
 
-    - run: ./build.sh release --skiptests
+    - run: ./build.sh release --test-suite=skip-all
       name: Release
 
     - name: Prepare Nuget

build/scripts/CommandLine.fs

@@ -9,15 +9,16 @@ open Microsoft.FSharp.Reflection
 open System
 open Bullseye
 
+type TestSuite = All | Unit | Integration | E2E | Skip_All | Skip_E2E 
+
 type Build =
     | [<CliPrefix(CliPrefix.None);SubCommand>] Clean
     | [<CliPrefix(CliPrefix.None);SubCommand>] Version
     | [<CliPrefix(CliPrefix.None);SubCommand>] Build
     | [<CliPrefix(CliPrefix.None);SubCommand>] Test
 
-    | [<CliPrefix(CliPrefix.None);SubCommand>] UnitTest 
-    | [<CliPrefix(CliPrefix.None);SubCommand>] Integrate
-    | [<CliPrefix(CliPrefix.None);SubCommand>] EndToEnd
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Unit_Test
+    | [<CliPrefix(CliPrefix.None);SubCommand>] End_To_End
 
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] PristineCheck 
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] GeneratePackages
@@ -27,10 +28,10 @@ type Build =
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] GenerateApiChanges 
     | [<CliPrefix(CliPrefix.None);SubCommand>] Release
 
-    | [<Inherit;AltCommandLine("-s")>] SingleTarget
+    | [<Inherit;AltCommandLine("-s")>] Single_Target
     | [<Inherit>] Token of string 
-    | [<Inherit;AltCommandLine("-c")>] SkipDirtyCheck
-    | [<Inherit;>] SkipTests
+    | [<Inherit;AltCommandLine("-c")>] Skip_Dirty_Check
+    | [<Inherit;>] Test_Suite of TestSuite 
 with
     interface IArgParserTemplate with
         member this.Usage =
@@ -40,11 +41,9 @@ with
             | Version -> "print version information"
             | Build -> "Run build"
 
-            | UnitTest  -> "build then run only the unit tests"
-            | Integrate -> "build then run only the integration tests"
-            | EndToEnd -> "build then run only the e2e tests"
-            | Test -> "build then run ALL the tests"
-
+            | Unit_Test -> "alias to providing: test --test-suite=unit"
+            | End_To_End -> "alias to providing: test --test-suite=e2e"
+            | Test -> "runs a clean build and then runs all the tests unless --test-suite is provided"
             | Release -> "runs build, tests, and create and validates the packages shy of publishing them"
 
             // steps
@@ -56,10 +55,11 @@ with
             | GenerateApiChanges -> "Undocumented, dependent target"
 
             // flags
-            | SingleTarget -> "Runs the provided sub command without running their dependencies"
+            | Single_Target -> "Runs the provided sub command without running their dependencies"
             | Token _ -> "Token to be used to authenticate with github"
-            | SkipDirtyCheck -> "Skip the clean checkout check that guards the release/publish targets"
-            | SkipTests -> "Skips running tests"
+            | Skip_Dirty_Check -> "Skip the clean checkout check that guards the release/publish targets"
+            | Test_Suite _ -> "Specify the test suite to run, defaults to all"
+
 
     member this.StepName =
         match FSharpValue.GetUnionFields(this, typeof<Build>) with
@@ -79,7 +79,7 @@ with
         Targets.Target(target.StepName, Action(fun _ -> action(parsed)))
 
     static member Cmd (dependsOn: Build list) (composedOf: Build list) action (target: Build) (parsed: ParseResults<Build>) =
-        let singleTarget = parsed.TryGetResult SingleTarget |> Option.isSome
+        let singleTarget = parsed.TryGetResult Single_Target |> Option.isSome
         let dependsOn = if singleTarget then [] else dependsOn
 
         let steps = dependsOn @ composedOf |> List.map (_.StepName)

build/scripts/Targets.fs

@@ -34,14 +34,12 @@ let private version _ =
 let private generatePackages _ = exec { run "dotnet" "pack" }
 
 let private pristineCheck (arguments:ParseResults<Build>) =
-    let skipCheck = arguments.TryGetResult SkipDirtyCheck |> Option.isSome
+    let skipCheck = arguments.TryGetResult Skip_Dirty_Check |> Option.isSome
     match skipCheck, Information.isCleanWorkingCopy "." with
     | true, _ -> printfn "Checkout is dirty but -c was specified to ignore this"
     | _, true  -> printfn "The checkout folder does not have pending changes, proceeding"
     | _ -> failwithf "The checkout folder has pending changes, aborting. Specify -c to ./build.sh to skip this check"
 
-type TestSuite = | Unit | Integration | E2E | All
-
 let private runTests suite _ =
     let logger =
         // use junit xml logging locally, github actions logs using console out formats
@@ -55,9 +53,11 @@ let private runTests suite _ =
     let filterArgs =
         match suite with
         | All -> []
-        | TestSuite.Unit ->  [ "--filter"; "FullyQualifiedName~.Tests" ]
-        | TestSuite.Integration -> [ "--filter"; "FullyQualifiedName~.IntegrationTests" ]
-        | TestSuite.E2E -> [ "--filter"; "FullyQualifiedName~.EndToEndTests" ]
+        | Skip_All -> ["--filter"; "FullyQualifiedName~.SKIPPING.ALL.TESTS"]
+        | Unit ->  [ "--filter"; "FullyQualifiedName~.Tests" ]
+        | Integration -> [ "--filter"; "FullyQualifiedName~.IntegrationTests" ]
+        | E2E -> [ "--filter"; "FullyQualifiedName~.EndToEndTests" ]
+        | Skip_E2E -> [ "--filter"; "FullyQualifiedName!~.EndToEndTests" ]
 
 
     let settingsArg = ["-s"; "tests/.runsettings"]
@@ -72,10 +72,14 @@ let private runTests suite _ =
         )
     }
 
-let private test suite (arguments:ParseResults<Build>) =
-    match arguments.TryGetResult SkipTests with
-    | None -> runTests suite arguments 
-    | Some _ -> printfn "Skipping tests because --skiptests was provided"
+let private test (arguments:ParseResults<Build>) =
+    let arg = arguments.TryGetResult Test_Suite
+    match arg with
+    | None -> runTests TestSuite.All arguments 
+    | Some suite ->
+        match suite with
+        | Skip_All -> printfn "Skipping tests because --test-suite skip was provided"
+        | _ -> runTests suite arguments   
 
 let private validateLicenses _ =
     let args = ["-u"; "-t"; "-i"; "Elastic.OpenTelemetry.sln"; "--use-project-assets-json"
@@ -149,10 +153,9 @@ let Setup (parsed:ParseResults<Build>) =
         | Clean -> Build.Cmd [Version] [] clean
         | Build -> Build.Cmd [Clean] [] build
 
-        | UnitTest -> Build.Cmd [Build] [] <| test TestSuite.Unit
-        | Integrate -> Build.Cmd [Build] [] <| test TestSuite.Integration
-        | EndToEnd -> Build.Cmd [Build] [] <| test TestSuite.E2E
-        | Test -> Build.Cmd [UnitTest; Integrate; EndToEnd] [] ignore
+        | End_To_End -> Build.Cmd [] [Build] <| runTests E2E
+        | Unit_Test -> Build.Cmd [] [Build] <| runTests Unit
+        | Test -> Build.Cmd [] [Build] test
 
         | Release -> 
             Build.Cmd 
@@ -169,10 +172,10 @@ let Setup (parsed:ParseResults<Build>) =
         | GenerateApiChanges -> Build.Step generateApiChanges
 
         // flags
-        | SingleTarget
-        | SkipTests
+        | Single_Target
+        | Test_Suite _
         | Token _
-        | SkipDirtyCheck -> Build.Ignore
+        | Skip_Dirty_Check -> Build.Ignore
 
     for target in Build.Targets do
         let setup = wireCommandLine target