CARL: issue-193 - Add Cleo release to Carls distribution and implemen…

…ted new endpoint to download tarball with cleo and set-env script. Co-authored-by: Anna Völker <[email protected]> Co-authored-by: Matthias Twardawski <[email protected]>
eclipse-opendut · May 6, 2024 · adcb860 · adcb860
1 parent 4bbccce
commit adcb860
Show file tree

Hide file tree

Showing 18 changed files with 444 additions and 39 deletions.
diff --git a/.ci/docker/carl/Dockerfile b/.ci/docker/carl/Dockerfile
@@ -14,7 +14,9 @@ COPY ./.ci/docker/carl/entrypoint.sh /opt/entrypoint.sh
 RUN groupadd --gid 1000 carl
 RUN useradd --create-home --uid 1000 --gid carl --shell /bin/bash carl
 
+ENTRYPOINT ["/opt/entrypoint.sh"]
+
+RUN chown -R carl:carl /opt/opendut-carl/
 USER carl
 
-ENTRYPOINT ["/opt/entrypoint.sh"]
 CMD ["/opt/opendut-carl/opendut-carl"]
diff --git a/.ci/docker/theo/src/commands/testenv.rs b/.ci/docker/theo/src/commands/testenv.rs
@@ -1,12 +1,13 @@
 use anyhow::Error;
 use clap::{ArgAction, Parser};
+use strum::IntoEnumIterator;
 
 use crate::commands::edgar::TestEdgarCli;
 use crate::core::dist::make_distribution_if_not_present;
 use crate::core::docker::{show_error_if_unhealthy_containers_were_found, start_netbird};
 use crate::core::docker::command::DockerCommand;
 use crate::core::docker::compose::{docker_compose_build, docker_compose_down, docker_compose_network_create, docker_compose_network_delete, docker_compose_up_expose_ports};
-use crate::core::docker::services::DockerCoreServices;
+use crate::core::docker::services::{DockerCoreServices};
 use crate::core::project::load_theo_environment_variables;
 
 /// Build and start test environment.
@@ -49,8 +50,8 @@ pub enum TaskCli {
 #[derive(Parser, Debug)]
 #[clap(version)]
 pub struct DestroyArgs {
-    #[clap(short = 's', long, default_value = "all")]
-    service: DockerCoreServices,
+    #[clap(short = 's', long)]
+    service: Option<DockerCoreServices>,
 }
 
 impl TestenvCli {
@@ -99,25 +100,27 @@ impl TestenvCli {
                 show_error_if_unhealthy_containers_were_found()?;
             }
             TaskCli::Destroy(service) => {
-                match service.service {
-                    DockerCoreServices::Network => { docker_compose_network_delete()?; }
-                    DockerCoreServices::Carl => { docker_compose_down(DockerCoreServices::Carl.as_str(), true)?; }
-                    DockerCoreServices::CarlOnHost => { docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true)?; }
-                    DockerCoreServices::Dev => { docker_compose_down(DockerCoreServices::Dev.as_str(), true)?; }
-                    DockerCoreServices::Keycloak => { docker_compose_down(DockerCoreServices::Keycloak.as_str(), true)?; }
-                    DockerCoreServices::Edgar => { docker_compose_down(DockerCoreServices::Edgar.as_str(), true)?; }
-                    DockerCoreServices::Netbird => { docker_compose_down(DockerCoreServices::Netbird.as_str(), true)?; }
-                    DockerCoreServices::Firefox => { docker_compose_down(DockerCoreServices::Firefox.as_str(), true)?; }
-                    DockerCoreServices::Telemetry => { docker_compose_down(DockerCoreServices::Telemetry.as_str(), true)?; }
-                    DockerCoreServices::All => {
-                        println!("Destroying all services.");
-                        docker_compose_down(DockerCoreServices::Firefox.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Edgar.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Carl.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Netbird.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Keycloak.as_str(), true)?;
-                        docker_compose_network_delete()?;
+                match &service.service {
+                    Some(service) => {
+                        match service {
+                            DockerCoreServices::Network => { docker_compose_network_delete() ?; }
+                            DockerCoreServices::Carl => { docker_compose_down(DockerCoreServices::Carl.as_str(), true) ?; }
+                            DockerCoreServices::CarlOnHost => { docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true) ?; }
+                            DockerCoreServices::Dev => { docker_compose_down(DockerCoreServices::Dev.as_str(), true) ?; }
+                            DockerCoreServices::Keycloak => { docker_compose_down(DockerCoreServices::Keycloak.as_str(), true) ?; }
+                            DockerCoreServices::Edgar => { docker_compose_down(DockerCoreServices::Edgar.as_str(), true) ?; }
+                            DockerCoreServices::Netbird => { docker_compose_down(DockerCoreServices::Netbird.as_str(), true) ?; }
+                            DockerCoreServices::Firefox => { docker_compose_down(DockerCoreServices::Firefox.as_str(), true) ?; }
+                            DockerCoreServices::Telemetry => { docker_compose_down(DockerCoreServices::Telemetry.as_str(), true) ?; }
+                        }
+                    }
+                    None => {
+                            println!("Destroying all services.");
+                            for docker_service in DockerCoreServices::iter() {
+                                docker_compose_down(docker_service.as_str(), true)?;
+                            }
+                            docker_compose_network_delete()?;
+
                     }
                 }
             }

diff --git a/.ci/docker/theo/src/core/docker/services.rs b/.ci/docker/theo/src/core/docker/services.rs
@@ -1,7 +1,7 @@
 use serde::Serialize;
 use strum::EnumIter;
 
-#[derive(Debug, Clone, clap::ValueEnum, Default, Serialize, EnumIter)]
+#[derive(Debug, Clone, clap::ValueEnum, Serialize, EnumIter)]
 pub(crate) enum DockerCoreServices {
     Network,
     Carl,
@@ -12,8 +12,6 @@ pub(crate) enum DockerCoreServices {
     Netbird,
     Firefox,
     Telemetry,
-    #[default]
-    All,
 }
 
 impl DockerCoreServices {
@@ -28,7 +26,6 @@ impl DockerCoreServices {
             DockerCoreServices::Network => "network",
             DockerCoreServices::Firefox => "firefox",
             DockerCoreServices::Telemetry => "telemetry",
-            DockerCoreServices::All => "all",
         }
     }
 }

diff --git a/.ci/xtask/src/core/types/arch.rs b/.ci/xtask/src/core/types/arch.rs
@@ -4,7 +4,7 @@ use clap::ValueEnum;
 use strum::IntoEnumIterator;
 
 /// General architecture used somewhere in the build process
-#[derive(Clone, Copy, Debug, strum::EnumIter)]
+#[derive(Clone, Copy, PartialEq, Debug, strum::EnumIter)]
 pub enum Arch {
     X86_64,
     Armhf,

diff --git a/.ci/xtask/src/packages/carl.rs b/.ci/xtask/src/packages/carl.rs
@@ -102,6 +102,7 @@ pub mod distribution {
 
         distribution::collect_executables(SELF_PACKAGE, target)?;
 
+        cleo::get_cleo(&distribution_out_dir)?;
         lea::get_lea(&distribution_out_dir)?;
         copy_license_json::copy_license_json(target, SkipGenerate::No)?;
 
@@ -112,6 +113,36 @@ pub mod distribution {
         Ok(())
     }
 
+    mod cleo {
+        use clap::ValueEnum;
+        use super::*;
+
+        #[tracing::instrument]
+        pub fn get_cleo(out_dir: &PathBuf) -> crate::Result {
+
+            let architectures = Arch::value_variants().iter()
+                .filter(|&&arch| arch != Arch::Wasm).collect::<Vec<_>>();
+
+            for arch in architectures {
+                crate::packages::cleo::build::build_release(arch.to_owned())?;
+                let cleo_build_dir = crate::packages::cleo::build::out_dir(arch.to_owned());
+
+                let cleo_out_dir = out_dir.join(Package::Cleo.ident());
+
+                fs::create_dir_all(&cleo_out_dir)?;
+
+                fs_extra::file::copy(
+                    cleo_build_dir,
+                    &cleo_out_dir.join(format!("{}-{}", Package::Cleo.ident(), arch.triple())),
+                    &fs_extra::file::CopyOptions::default()
+                        .overwrite(true)
+                )?;
+            }
+
+            Ok(())
+        }
+    }
+
     mod lea {
         use super::*;
 
@@ -151,7 +182,7 @@ pub mod distribution {
             match skip_generate {
                 SkipGenerate::Yes => info!("Skipping generation of licenses, as requested. Directly attempting to copy to target location."),
                 SkipGenerate::No => {
-                    for package in [SELF_PACKAGE, Package::Lea, Package::Edgar] {
+                    for package in [SELF_PACKAGE, Package::Lea, Package::Edgar, Package::Cleo] {
                         crate::tasks::licenses::json::export_json(package)?;
                     }
                 }
@@ -161,13 +192,16 @@ pub mod distribution {
             let carl_out_file = crate::tasks::distribution::copy_license_json::out_file(SELF_PACKAGE, target);
             let out_dir = carl_out_file.parent().unwrap();
 
+            let cleo_in_file = crate::tasks::licenses::json::out_file(Package::Cleo);
+            let cleo_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Cleo));
             let lea_in_file = crate::tasks::licenses::json::out_file(Package::Lea);
             let lea_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Lea));
             let edgar_in_file = crate::tasks::licenses::json::out_file(Package::Edgar);
             let edgar_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Edgar));
 
             fs::create_dir_all(out_dir)?;
             fs::copy(carl_in_file, &carl_out_file)?;
+            fs::copy(cleo_in_file, &cleo_out_file)?;
             fs::copy(lea_in_file, &lea_out_file)?;
             fs::copy(edgar_in_file, &edgar_out_file)?;
 
@@ -176,6 +210,7 @@ pub mod distribution {
                 json!({
                     "carl": carl_out_file.file_name().unwrap().to_str(),
                     "edgar": edgar_out_file.file_name().unwrap().to_str(),
+                    "cleo": cleo_out_file.file_name().unwrap().to_str(),
                     "lea": lea_out_file.file_name().unwrap().to_str(),
                 }).to_string(),
             )?;
@@ -213,36 +248,41 @@ pub mod distribution {
             carl_dir.assert(path::is_dir());
 
             let opendut_carl_executable = carl_dir.child(SELF_PACKAGE.ident());
+            let opendut_cleo_dir = carl_dir.child(Package::Cleo.ident());
             let opendut_lea_dir = carl_dir.child(Package::Lea.ident());
             let licenses_dir = carl_dir.child("licenses");
 
             carl_dir.dir_contains_exactly_in_order(vec![
                 &licenses_dir,
                 &opendut_carl_executable,
+                &opendut_cleo_dir,
                 &opendut_lea_dir,
             ]);
 
             opendut_carl_executable.assert_non_empty_file();
+            opendut_cleo_dir.assert(path::is_dir());
             opendut_lea_dir.assert(path::is_dir());
             licenses_dir.assert(path::is_dir());
 
             { //validate license dir contents
                 let licenses_index_file = licenses_dir.child("index.json");
                 let licenses_carl_file = licenses_dir.child("opendut-carl.licenses.json");
                 let licenses_edgar_file = licenses_dir.child("opendut-edgar.licenses.json");
+                let licenses_cleo_file = licenses_dir.child("opendut-cleo.licenses.json");
                 let licenses_lea_file = licenses_dir.child("opendut-lea.licenses.json");
 
                 licenses_dir.dir_contains_exactly_in_order(vec![
                     &licenses_index_file,
                     &licenses_carl_file,
+                    &licenses_cleo_file,
                     &licenses_edgar_file,
                     &licenses_lea_file,
                 ]);
 
                 licenses_index_file.assert(path::is_file());
                 let licenses_index_content = fs::read_to_string(licenses_index_file)?;
 
-                for license_file in [&licenses_edgar_file, &licenses_carl_file, &licenses_lea_file] {
+                for license_file in [&licenses_edgar_file, &licenses_carl_file, &licenses_cleo_file, &licenses_lea_file] {
                     assert!(
                         licenses_index_content.contains(license_file.file_name_str()),
                         "The license index.json did not contain entry for expected file: {}", license_file.display()

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -80,7 +80,7 @@ jobs:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"
 
   bundle-carl:
-    needs: [ legal, build-carl, build-lea ]
+    needs: [ legal, build-carl, build-lea, build-cleo ]
     uses: ./.github/workflows/job-bundle-carl.yaml
     with:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"

diff --git a/.github/workflows/job-bundle-carl.yaml b/.github/workflows/job-bundle-carl.yaml
@@ -43,6 +43,11 @@ jobs:
         with:
           name: "${{ matrix.package.name }}-${{ matrix.package.target }}-${{ github.sha }}"
           path: "./target/ci/distribution/${{ matrix.package.target }}/${{ matrix.package.name }}/"
+      - name: Download opendut-cleo
+        uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935
+        with:
+          pattern: "opendut-cleo-*-${{ github.sha }}"
+          path: "./target/ci/distribution/${{ matrix.package.target }}/${{ matrix.package.name }}/opendut-cleo"
       - name: Download opendut-lea
         uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935
         with:

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -66,7 +66,7 @@ jobs:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"
 
   bundle-carl:
-    needs: [ legal, build-carl, build-lea ]
+    needs: [ legal, build-carl, build-lea, bundle-cleo ]
     uses: ./.github/workflows/job-bundle-carl.yaml
     with:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/doc/src/user-manual/cleo/setup.md b/doc/src/user-manual/cleo/setup.md
@@ -10,5 +10,49 @@ The possible configuration values and their defaults can be seen here:
 {{#include ../../../../opendut-cleo/cleo.toml}}
 ```
 
+## Download CLEO from CARL
+It is also possible to download CLEO from one of CARLs endpoints. The downloaded file contains the binary for CLEO for the requested architecture,
+the necessary certificate file, as well as a setup script.
+
+The archive can be requested at `https://{CARL-HOST}/api/cleo/{architecture}/download`.
+
+Available architectures are:
+- x86_64-unknown-linux-gnu
+- armv7-unknown-linux-gnueabihf
+- aarch64-unknown-linux-gnu
+
+This might be the go-to way, if you want to use CLEO in your pipeline. 
+Once downloaded, extract the files with the command `tar xvf opendut-cleo-{architecture}.tar.gz`. It will then be extracted into
+the folder which is the current work directory. You might want to use another directory of your choice.
+The tarball contains the `set-env-var.sh` shell script. It can be executed by the command `source set-env-var.sh`, which then sets the
+following environment variables to run CLEO:
+````
+OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SCOPES
+OPENDUT_CLEO_NETWORK_TLS_DOMAIN_NAME_OVERRIDE
+OPENDUT_CLEO_NETWORK_TLS_CA
+OPENDUT_CLEO_NETWORK_CARL_HOST
+OPENDUT_CLEO_NETWORK_CARL_PORT
+OPENDUT_CLEO_NETWORK_OIDC_ENABLED
+OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ISSUER_URL
+SSL_CERT_FILE
+````
+
+`SSL_CERT_FILE` is a mandatory environment variable for the current state of the implementation and has the same value as the 
+`OPENDUT_CLEO_NETWORK_TLS_CA`. This might change in the future. 
+
+The script will not set the environment variables for CLIENT_ID and CLIENT_SECRET. This has to be done by the users themselves.
+This can easily be done by entering the following commands:
+````
+export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ID={{ CLIENT ID VARIABLE }} 
+export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SECRET={{ CLIENT SECRET VARIABLE }} 
+````
+These two variables can be obtained by logging in to Keycloak.
+
+### TL;DR
+1. Download archive from `https://{CARL-HOST}/api/cleo/{architecture}/download`
+2. Extract `tar xvf opendut-cleo-{architecture}.tar.gz`
+3. Execute `source set-env-var.sh`
+4. Add two environment variable `export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ID={{ CLIENT ID VARIABLE }}` and `export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SECRET={{ CLIENT SECRET VARIABLE }}`
+
 ## Additional notes
 - The CA certificate to be provided for CLEO depends on the used certificate authority used on server side for CARL.
diff --git a/opendut-carl/Cargo.toml b/opendut-carl/Cargo.toml
@@ -18,6 +18,7 @@ axum = { workspace = true }
 axum-server = { workspace = true, features = ["tls-rustls"] }
 axum-server-dual-protocol = { workspace = true }
 config = { workspace = true }
+flate2 = { workspace = true }
 futures = { workspace = true }
 googletest = { workspace = true }
 http = { workspace = true }
@@ -29,6 +30,7 @@ opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 shadow-rs = { workspace = true, default-features = true }
+tar = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tokio-stream = { workspace = true, features = ["full"] }
@@ -42,6 +44,7 @@ url = { workspace = true, features = ["serde"] }
 uuid = { workspace = true }
 
 [dev-dependencies]
+assert_fs = { workspace = true }
 async-trait = { workspace = true }
 rstest = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }