CARL: issue-193 - Add Cleo release to Carls distribution and implemen…

…ted new endpoint to download tarball with cleo and set-env script. Co-authored-by: Anna Völker <[email protected]> Co-authored-by: Matthias Twardawski <[email protected]>
eclipse-opendut · Apr 22, 2024 · 3c52199 · 3c52199
1 parent 4f0c9c4
commit 3c52199
Show file tree

Hide file tree

Showing 10 changed files with 246 additions and 16 deletions.
diff --git a/.ci/docker/theo/src/commands/testenv.rs b/.ci/docker/theo/src/commands/testenv.rs
@@ -1,12 +1,13 @@
 use anyhow::Error;
 use clap::{ArgAction, Parser};
+use strum::IntoEnumIterator;
 
 use crate::commands::edgar::TestEdgarCli;
 use crate::core::dist::make_distribution_if_not_present;
 use crate::core::docker::{show_error_if_unhealthy_containers_were_found, start_netbird};
 use crate::core::docker::command::DockerCommand;
 use crate::core::docker::compose::{docker_compose_build, docker_compose_down, docker_compose_network_create, docker_compose_network_delete, docker_compose_up_expose_ports};
-use crate::core::docker::services::DockerCoreServices;
+use crate::core::docker::services::{DockerCoreServices};
 use crate::core::project::load_theo_environment_variables;
 
 /// Build and start test environment.
@@ -111,12 +112,9 @@ impl TestenvCli {
                     DockerCoreServices::Telemetry => { docker_compose_down(DockerCoreServices::Telemetry.as_str(), true)?; }
                     DockerCoreServices::All => {
                         println!("Destroying all services.");
-                        docker_compose_down(DockerCoreServices::Firefox.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Edgar.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Carl.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Netbird.as_str(), true)?;
-                        docker_compose_down(DockerCoreServices::Keycloak.as_str(), true)?;
+                        for docker_service in DockerCoreServices::iter() {
+                            docker_compose_down(docker_service.as_str(), true)?;
+                        }
                         docker_compose_network_delete()?;
                     }
                 }

diff --git a/.ci/xtask/src/core/types/arch.rs b/.ci/xtask/src/core/types/arch.rs
@@ -4,7 +4,7 @@ use clap::ValueEnum;
 use strum::IntoEnumIterator;
 
 /// General architecture used somewhere in the build process
-#[derive(Clone, Copy, Debug, strum::EnumIter)]
+#[derive(Clone, Copy, PartialEq, Debug, strum::EnumIter)]
 pub enum Arch {
     X86_64,
     Armhf,

diff --git a/.ci/xtask/src/packages/carl.rs b/.ci/xtask/src/packages/carl.rs
@@ -102,6 +102,7 @@ pub mod distribution {
 
         distribution::collect_executables(SELF_PACKAGE, target)?;
 
+        cleo::get_cleo(&distribution_out_dir)?;
         lea::get_lea(&distribution_out_dir)?;
         copy_license_json::copy_license_json(target, SkipGenerate::No)?;
 
@@ -112,6 +113,36 @@ pub mod distribution {
         Ok(())
     }
 
+    mod cleo {
+        use clap::ValueEnum;
+        use super::*;
+
+        #[tracing::instrument]
+        pub fn get_cleo(out_dir: &PathBuf) -> crate::Result {
+
+            let architectures = Arch::value_variants().iter()
+                .filter(|&&arch| arch != Arch::Wasm).collect::<Vec<_>>();
+
+            for arch in architectures {
+                crate::packages::cleo::build::build_release(arch.to_owned())?;
+                let cleo_build_dir = crate::packages::cleo::build::out_dir(arch.to_owned());
+
+                let cleo_out_dir = out_dir.join(Package::Cleo.ident());
+
+                fs::create_dir_all(&cleo_out_dir)?;
+
+                fs_extra::file::copy(
+                    cleo_build_dir,
+                    &cleo_out_dir.join(format!("{}-{}", Package::Cleo.ident(), arch.triple())),
+                    &fs_extra::file::CopyOptions::default()
+                        .overwrite(true)
+                )?;
+            }
+
+            Ok(())
+        }
+    }
+
     mod lea {
         use super::*;
 
@@ -151,7 +182,7 @@ pub mod distribution {
             match skip_generate {
                 SkipGenerate::Yes => info!("Skipping generation of licenses, as requested. Directly attempting to copy to target location."),
                 SkipGenerate::No => {
-                    for package in [SELF_PACKAGE, Package::Lea, Package::Edgar] {
+                    for package in [SELF_PACKAGE, Package::Lea, Package::Edgar, Package::Cleo] {
                         crate::tasks::licenses::json::export_json(package)?;
                     }
                 }
@@ -161,13 +192,16 @@ pub mod distribution {
             let carl_out_file = crate::tasks::distribution::copy_license_json::out_file(SELF_PACKAGE, target);
             let out_dir = carl_out_file.parent().unwrap();
 
+            let cleo_in_file = crate::tasks::licenses::json::out_file(Package::Cleo);
+            let cleo_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Cleo));
             let lea_in_file = crate::tasks::licenses::json::out_file(Package::Lea);
             let lea_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Lea));
             let edgar_in_file = crate::tasks::licenses::json::out_file(Package::Edgar);
             let edgar_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Edgar));
 
             fs::create_dir_all(out_dir)?;
             fs::copy(carl_in_file, &carl_out_file)?;
+            fs::copy(cleo_in_file, &cleo_out_file)?;
             fs::copy(lea_in_file, &lea_out_file)?;
             fs::copy(edgar_in_file, &edgar_out_file)?;
 
@@ -176,6 +210,7 @@ pub mod distribution {
                 json!({
                     "carl": carl_out_file.file_name().unwrap().to_str(),
                     "edgar": edgar_out_file.file_name().unwrap().to_str(),
+                    "cleo": cleo_out_file.file_name().unwrap().to_str(),
                     "lea": lea_out_file.file_name().unwrap().to_str(),
                 }).to_string(),
             )?;
@@ -213,36 +248,41 @@ pub mod distribution {
             carl_dir.assert(path::is_dir());
 
             let opendut_carl_executable = carl_dir.child(SELF_PACKAGE.ident());
+            let opendut_cleo_dir = carl_dir.child(Package::Cleo.ident());
             let opendut_lea_dir = carl_dir.child(Package::Lea.ident());
             let licenses_dir = carl_dir.child("licenses");
 
             carl_dir.dir_contains_exactly_in_order(vec![
                 &licenses_dir,
                 &opendut_carl_executable,
+                &opendut_cleo_dir,
                 &opendut_lea_dir,
             ]);
 
             opendut_carl_executable.assert_non_empty_file();
+            opendut_cleo_dir.assert(path::is_dir());
             opendut_lea_dir.assert(path::is_dir());
             licenses_dir.assert(path::is_dir());
 
             { //validate license dir contents
                 let licenses_index_file = licenses_dir.child("index.json");
                 let licenses_carl_file = licenses_dir.child("opendut-carl.licenses.json");
                 let licenses_edgar_file = licenses_dir.child("opendut-edgar.licenses.json");
+                let licenses_cleo_file = licenses_dir.child("opendut-cleo.licenses.json");
                 let licenses_lea_file = licenses_dir.child("opendut-lea.licenses.json");
 
                 licenses_dir.dir_contains_exactly_in_order(vec![
                     &licenses_index_file,
                     &licenses_carl_file,
+                    &licenses_cleo_file,
                     &licenses_edgar_file,
                     &licenses_lea_file,
                 ]);
 
                 licenses_index_file.assert(path::is_file());
                 let licenses_index_content = fs::read_to_string(licenses_index_file)?;
 
-                for license_file in [&licenses_edgar_file, &licenses_carl_file, &licenses_lea_file] {
+                for license_file in [&licenses_edgar_file, &licenses_carl_file, &licenses_cleo_file, &licenses_lea_file] {
                     assert!(
                         licenses_index_content.contains(license_file.file_name_str()),
                         "The license index.json did not contain entry for expected file: {}", license_file.display()

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -80,7 +80,7 @@ jobs:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"
 
   bundle-carl:
-    needs: [ legal, build-carl, build-lea ]
+    needs: [ legal, build-carl, build-lea, build-cleo ]
     uses: ./.github/workflows/job-bundle-carl.yaml
     with:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"

diff --git a/.github/workflows/job-bundle-carl.yaml b/.github/workflows/job-bundle-carl.yaml
@@ -43,6 +43,11 @@ jobs:
         with:
           name: "${{ matrix.package.name }}-${{ matrix.package.target }}-${{ github.sha }}"
           path: "./target/ci/distribution/${{ matrix.package.target }}/${{ matrix.package.name }}/"
+      - name: Download opendut-cleo
+        uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935
+        with:
+          pattern: "opendut-cleo-*-${{ github.sha }}"
+          path: "./target/ci/distribution/${{ matrix.package.target }}/${{ matrix.package.name }}/opendut-cleo"
       - name: Download opendut-lea
         uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935
         with:

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -66,7 +66,7 @@ jobs:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"
 
   bundle-carl:
-    needs: [ legal, build-carl, build-lea ]
+    needs: [ legal, build-carl, build-lea, bundle-cleo ]
     uses: ./.github/workflows/job-bundle-carl.yaml
     with:
       runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}"

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/doc/src/user-manual/cleo/install.md b/doc/src/user-manual/cleo/install.md
@@ -25,5 +25,51 @@ scopes = "openid,profile,email,roles,groups"
 secret = "<tbd>"
 ````
 
+## Download CLEO from CARL
+It is also possible to download CLEO from one of CARLs endpoints. The downloaded file contains the binary for CLEO for the requested architecture
+as well as a script.
+
+The archive can be requested at `https://{CARL-HOST}/api/cleo/{architecture}/download`.
+
+This might be the go-to way, if you want to use CLEO in your pipeline. 
+Once downloaded, extract the files with the command `tar -xvf opendut-cleo-{architecture}.tar.gz`. It will then be extracted into
+the folder which is the current work directory. You might want to use another directory of your choice.
+The tarball contains the `set-env-var.sh` shell script. It can be executed by the command `source set-env-var.sh`, which then sets the 
+following environment variables to run CLEO: 
+````
+OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SCOPES
+OPENDUT_CLEO_NETWORK_TLS_DOMAIN_NAME_OVERRIDE
+OPENDUT_CLEO_NETWORK_TLS_CA
+OPENDUT_CLEO_NETWORK_CARL_HOST
+OPENDUT_CLEO_NETWORK_CARL_PORT
+OPENDUT_CLEO_NETWORK_OIDC_ENABLED
+OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ISSUER_URL
+````
+
+The script will not set the environment variables for CLIENT_ID and CLIENT_SECRET. This has to be done by the users themselves.
+This can easily be done by entering the following commands:
+````
+export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ID={{ CLIENT ID VARIBALE }} 
+export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SECRET={{ CLIENT SECRET VARIBALE }} 
+````
+These two variables can be obtained by logging in to Keycloak.
+
+The last thing to do, is to check if there is already a certificate, which is required to use CLEO.
+The provided script is expecting a certificate at the location `/etc/opendut/tls`. If there is no such folder or no file, 
+it has to be created manually. In this case execute the command: `mkdir -p /etc/opendut/tls`. This will create a folder,
+if not yet existent. After that copy the correct certificate (most likely the CA certificate from keycloak) into the newly 
+created folder with `cp {source folder}/cert.pem /etc/opendut/tls/ca.pem`. The naming is important, because this is how it was stored 
+in the environment variable `OPENDUT_CLEO_NETWORK_TLS_CA`.
+
+
+### TL;DR
+1. Download archive from `https://{CARL-HOST}/api/cleo/{architecture}/download`
+2. Extract `tar -xvf opendut-cleo-{architecture}.tar.gz`
+3. Execute `source set-env-var.sh`
+4. Add two environment variable `export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ID={{ CLIENT ID VARIBALE }}` and `export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SECRET={{ CLIENT SECRET VARIBALE }}`
+5. (Optional) Check if certificate exists at `/etc/opendut/tls` otherwise follow these steps:
+   1. `mkdir -p /etc/opendut/tls`
+   2. `cp {source folder}/cert.pem /etc/opendut/tls/ca.pem`
+
 ## Additional notes
 - The ca certificate to be provided for CLEO depends on the used certificate authority used on server side for CARL
diff --git a/opendut-carl/Cargo.toml b/opendut-carl/Cargo.toml
@@ -18,6 +18,7 @@ axum = { workspace = true }
 axum-server = { workspace = true, features = ["tls-rustls"] }
 axum-server-dual-protocol = { workspace = true }
 config = { workspace = true }
+flate2 = { workspace = true }
 futures = { workspace = true }
 googletest = { workspace = true }
 http = { workspace = true }
@@ -29,6 +30,7 @@ opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 shadow-rs = { workspace = true, default-features = true }
+tar = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tokio-stream = { workspace = true, features = ["full"] }