diff --git a/.ci/docker/carl/Dockerfile b/.ci/docker/carl/Dockerfile index bf7dd94c6..c2f04b359 100644 --- a/.ci/docker/carl/Dockerfile +++ b/.ci/docker/carl/Dockerfile @@ -14,7 +14,9 @@ COPY ./.ci/docker/carl/entrypoint.sh /opt/entrypoint.sh RUN groupadd --gid 1000 carl RUN useradd --create-home --uid 1000 --gid carl --shell /bin/bash carl +ENTRYPOINT ["/opt/entrypoint.sh"] + +RUN chown -R carl:carl /opt/opendut-carl/ USER carl -ENTRYPOINT ["/opt/entrypoint.sh"] CMD ["/opt/opendut-carl/opendut-carl"] diff --git a/.ci/docker/theo/src/commands/testenv.rs b/.ci/docker/theo/src/commands/testenv.rs index a4de8b72a..29d19f078 100644 --- a/.ci/docker/theo/src/commands/testenv.rs +++ b/.ci/docker/theo/src/commands/testenv.rs @@ -1,12 +1,13 @@ use anyhow::Error; use clap::{ArgAction, Parser}; +use strum::IntoEnumIterator; use crate::commands::edgar::TestEdgarCli; use crate::core::dist::make_distribution_if_not_present; use crate::core::docker::{show_error_if_unhealthy_containers_were_found, start_netbird}; use crate::core::docker::command::DockerCommand; use crate::core::docker::compose::{docker_compose_build, docker_compose_down, docker_compose_network_create, docker_compose_network_delete, docker_compose_up_expose_ports}; -use crate::core::docker::services::DockerCoreServices; +use crate::core::docker::services::{DockerCoreServices}; use crate::core::project::load_theo_environment_variables; /// Build and start test environment. @@ -49,8 +50,8 @@ pub enum TaskCli { #[derive(Parser, Debug)] #[clap(version)] pub struct DestroyArgs { - #[clap(short = 's', long, default_value = "all")] - service: DockerCoreServices, + #[clap(short = 's', long)] + service: Option, } impl TestenvCli { @@ -99,25 +100,27 @@ impl TestenvCli { show_error_if_unhealthy_containers_were_found()?; } TaskCli::Destroy(service) => { - match service.service { - DockerCoreServices::Network => { docker_compose_network_delete()?; } - DockerCoreServices::Carl => { docker_compose_down(DockerCoreServices::Carl.as_str(), true)?; } - DockerCoreServices::CarlOnHost => { docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true)?; } - DockerCoreServices::Dev => { docker_compose_down(DockerCoreServices::Dev.as_str(), true)?; } - DockerCoreServices::Keycloak => { docker_compose_down(DockerCoreServices::Keycloak.as_str(), true)?; } - DockerCoreServices::Edgar => { docker_compose_down(DockerCoreServices::Edgar.as_str(), true)?; } - DockerCoreServices::Netbird => { docker_compose_down(DockerCoreServices::Netbird.as_str(), true)?; } - DockerCoreServices::Firefox => { docker_compose_down(DockerCoreServices::Firefox.as_str(), true)?; } - DockerCoreServices::Telemetry => { docker_compose_down(DockerCoreServices::Telemetry.as_str(), true)?; } - DockerCoreServices::All => { - println!("Destroying all services."); - docker_compose_down(DockerCoreServices::Firefox.as_str(), true)?; - docker_compose_down(DockerCoreServices::Edgar.as_str(), true)?; - docker_compose_down(DockerCoreServices::Carl.as_str(), true)?; - docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true)?; - docker_compose_down(DockerCoreServices::Netbird.as_str(), true)?; - docker_compose_down(DockerCoreServices::Keycloak.as_str(), true)?; - docker_compose_network_delete()?; + match &service.service { + Some(service) => { + match service { + DockerCoreServices::Network => { docker_compose_network_delete() ?; } + DockerCoreServices::Carl => { docker_compose_down(DockerCoreServices::Carl.as_str(), true) ?; } + DockerCoreServices::CarlOnHost => { docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true) ?; } + DockerCoreServices::Dev => { docker_compose_down(DockerCoreServices::Dev.as_str(), true) ?; } + DockerCoreServices::Keycloak => { docker_compose_down(DockerCoreServices::Keycloak.as_str(), true) ?; } + DockerCoreServices::Edgar => { docker_compose_down(DockerCoreServices::Edgar.as_str(), true) ?; } + DockerCoreServices::Netbird => { docker_compose_down(DockerCoreServices::Netbird.as_str(), true) ?; } + DockerCoreServices::Firefox => { docker_compose_down(DockerCoreServices::Firefox.as_str(), true) ?; } + DockerCoreServices::Telemetry => { docker_compose_down(DockerCoreServices::Telemetry.as_str(), true) ?; } + } + } + None => { + println!("Destroying all services."); + for docker_service in DockerCoreServices::iter() { + docker_compose_down(docker_service.as_str(), true)?; + } + docker_compose_network_delete()?; + } } } diff --git a/.ci/docker/theo/src/core/docker/services.rs b/.ci/docker/theo/src/core/docker/services.rs index c3415712a..b2e35a463 100644 --- a/.ci/docker/theo/src/core/docker/services.rs +++ b/.ci/docker/theo/src/core/docker/services.rs @@ -1,7 +1,7 @@ use serde::Serialize; use strum::EnumIter; -#[derive(Debug, Clone, clap::ValueEnum, Default, Serialize, EnumIter)] +#[derive(Debug, Clone, clap::ValueEnum, Serialize, EnumIter)] pub(crate) enum DockerCoreServices { Network, Carl, @@ -12,8 +12,6 @@ pub(crate) enum DockerCoreServices { Netbird, Firefox, Telemetry, - #[default] - All, } impl DockerCoreServices { @@ -28,7 +26,6 @@ impl DockerCoreServices { DockerCoreServices::Network => "network", DockerCoreServices::Firefox => "firefox", DockerCoreServices::Telemetry => "telemetry", - DockerCoreServices::All => "all", } } } diff --git a/.ci/xtask/src/core/types/arch.rs b/.ci/xtask/src/core/types/arch.rs index 1334a4b08..33caa2a0c 100644 --- a/.ci/xtask/src/core/types/arch.rs +++ b/.ci/xtask/src/core/types/arch.rs @@ -4,7 +4,7 @@ use clap::ValueEnum; use strum::IntoEnumIterator; /// General architecture used somewhere in the build process -#[derive(Clone, Copy, Debug, strum::EnumIter)] +#[derive(Clone, Copy, PartialEq, Debug, strum::EnumIter)] pub enum Arch { X86_64, Armhf, diff --git a/.ci/xtask/src/packages/carl.rs b/.ci/xtask/src/packages/carl.rs index 3da06516a..65dc50fed 100644 --- a/.ci/xtask/src/packages/carl.rs +++ b/.ci/xtask/src/packages/carl.rs @@ -102,6 +102,7 @@ pub mod distribution { distribution::collect_executables(SELF_PACKAGE, target)?; + cleo::get_cleo(&distribution_out_dir)?; lea::get_lea(&distribution_out_dir)?; copy_license_json::copy_license_json(target, SkipGenerate::No)?; @@ -112,6 +113,36 @@ pub mod distribution { Ok(()) } + mod cleo { + use clap::ValueEnum; + use super::*; + + #[tracing::instrument] + pub fn get_cleo(out_dir: &PathBuf) -> crate::Result { + + let architectures = Arch::value_variants().iter() + .filter(|&&arch| arch != Arch::Wasm).collect::>(); + + for arch in architectures { + crate::packages::cleo::build::build_release(arch.to_owned())?; + let cleo_build_dir = crate::packages::cleo::build::out_dir(arch.to_owned()); + + let cleo_out_dir = out_dir.join(Package::Cleo.ident()); + + fs::create_dir_all(&cleo_out_dir)?; + + fs_extra::file::copy( + cleo_build_dir, + &cleo_out_dir.join(format!("{}-{}", Package::Cleo.ident(), arch.triple())), + &fs_extra::file::CopyOptions::default() + .overwrite(true) + )?; + } + + Ok(()) + } + } + mod lea { use super::*; @@ -151,7 +182,7 @@ pub mod distribution { match skip_generate { SkipGenerate::Yes => info!("Skipping generation of licenses, as requested. Directly attempting to copy to target location."), SkipGenerate::No => { - for package in [SELF_PACKAGE, Package::Lea, Package::Edgar] { + for package in [SELF_PACKAGE, Package::Lea, Package::Edgar, Package::Cleo] { crate::tasks::licenses::json::export_json(package)?; } } @@ -161,6 +192,8 @@ pub mod distribution { let carl_out_file = crate::tasks::distribution::copy_license_json::out_file(SELF_PACKAGE, target); let out_dir = carl_out_file.parent().unwrap(); + let cleo_in_file = crate::tasks::licenses::json::out_file(Package::Cleo); + let cleo_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Cleo)); let lea_in_file = crate::tasks::licenses::json::out_file(Package::Lea); let lea_out_file = out_dir.join(crate::tasks::licenses::json::out_file_name(Package::Lea)); let edgar_in_file = crate::tasks::licenses::json::out_file(Package::Edgar); @@ -168,6 +201,7 @@ pub mod distribution { fs::create_dir_all(out_dir)?; fs::copy(carl_in_file, &carl_out_file)?; + fs::copy(cleo_in_file, &cleo_out_file)?; fs::copy(lea_in_file, &lea_out_file)?; fs::copy(edgar_in_file, &edgar_out_file)?; @@ -176,6 +210,7 @@ pub mod distribution { json!({ "carl": carl_out_file.file_name().unwrap().to_str(), "edgar": edgar_out_file.file_name().unwrap().to_str(), + "cleo": cleo_out_file.file_name().unwrap().to_str(), "lea": lea_out_file.file_name().unwrap().to_str(), }).to_string(), )?; @@ -213,16 +248,19 @@ pub mod distribution { carl_dir.assert(path::is_dir()); let opendut_carl_executable = carl_dir.child(SELF_PACKAGE.ident()); + let opendut_cleo_dir = carl_dir.child(Package::Cleo.ident()); let opendut_lea_dir = carl_dir.child(Package::Lea.ident()); let licenses_dir = carl_dir.child("licenses"); carl_dir.dir_contains_exactly_in_order(vec![ &licenses_dir, &opendut_carl_executable, + &opendut_cleo_dir, &opendut_lea_dir, ]); opendut_carl_executable.assert_non_empty_file(); + opendut_cleo_dir.assert(path::is_dir()); opendut_lea_dir.assert(path::is_dir()); licenses_dir.assert(path::is_dir()); @@ -230,11 +268,13 @@ pub mod distribution { let licenses_index_file = licenses_dir.child("index.json"); let licenses_carl_file = licenses_dir.child("opendut-carl.licenses.json"); let licenses_edgar_file = licenses_dir.child("opendut-edgar.licenses.json"); + let licenses_cleo_file = licenses_dir.child("opendut-cleo.licenses.json"); let licenses_lea_file = licenses_dir.child("opendut-lea.licenses.json"); licenses_dir.dir_contains_exactly_in_order(vec![ &licenses_index_file, &licenses_carl_file, + &licenses_cleo_file, &licenses_edgar_file, &licenses_lea_file, ]); @@ -242,7 +282,7 @@ pub mod distribution { licenses_index_file.assert(path::is_file()); let licenses_index_content = fs::read_to_string(licenses_index_file)?; - for license_file in [&licenses_edgar_file, &licenses_carl_file, &licenses_lea_file] { + for license_file in [&licenses_edgar_file, &licenses_carl_file, &licenses_cleo_file, &licenses_lea_file] { assert!( licenses_index_content.contains(license_file.file_name_str()), "The license index.json did not contain entry for expected file: {}", license_file.display() diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 901b9004e..cab2a4968 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -80,7 +80,7 @@ jobs: runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}" bundle-carl: - needs: [ legal, build-carl, build-lea ] + needs: [ legal, build-carl, build-lea, build-cleo ] uses: ./.github/workflows/job-bundle-carl.yaml with: runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}" diff --git a/.github/workflows/job-bundle-carl.yaml b/.github/workflows/job-bundle-carl.yaml index fcff21700..b6777e2de 100644 --- a/.github/workflows/job-bundle-carl.yaml +++ b/.github/workflows/job-bundle-carl.yaml @@ -43,6 +43,11 @@ jobs: with: name: "${{ matrix.package.name }}-${{ matrix.package.target }}-${{ github.sha }}" path: "./target/ci/distribution/${{ matrix.package.target }}/${{ matrix.package.name }}/" + - name: Download opendut-cleo + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 + with: + pattern: "opendut-cleo-*-${{ github.sha }}" + path: "./target/ci/distribution/${{ matrix.package.target }}/${{ matrix.package.name }}/opendut-cleo" - name: Download opendut-lea uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2efbe115b..aefcc5b75 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -66,7 +66,7 @@ jobs: runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}" bundle-carl: - needs: [ legal, build-carl, build-lea ] + needs: [ legal, build-carl, build-lea, bundle-cleo ] uses: ./.github/workflows/job-bundle-carl.yaml with: runs-on: "${{ vars.OPENDUT_GH_RUNNER_LARGE || '[\"ubuntu-latest\"]' }}" diff --git a/Cargo.lock b/Cargo.lock index d984ea657..46846f366 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2590,11 +2590,13 @@ name = "opendut-carl" version = "0.1.0" dependencies = [ "anyhow", + "assert_fs", "async-trait", "axum", "axum-server", "axum-server-dual-protocol", "config", + "flate2", "futures", "googletest", "http 0.2.12", @@ -2612,6 +2614,7 @@ dependencies = [ "rstest", "serde", "shadow-rs", + "tar", "thiserror", "tokio", "tokio-stream", diff --git a/doc/src/user-manual/cleo/setup.md b/doc/src/user-manual/cleo/setup.md index fe681972a..815070576 100644 --- a/doc/src/user-manual/cleo/setup.md +++ b/doc/src/user-manual/cleo/setup.md @@ -10,5 +10,49 @@ The possible configuration values and their defaults can be seen here: {{#include ../../../../opendut-cleo/cleo.toml}} ``` +## Download CLEO from CARL +It is also possible to download CLEO from one of CARLs endpoints. The downloaded file contains the binary for CLEO for the requested architecture, +the necessary certificate file, as well as a setup script. + +The archive can be requested at `https://{CARL-HOST}/api/cleo/{architecture}/download`. + +Available architectures are: +- x86_64-unknown-linux-gnu +- armv7-unknown-linux-gnueabihf +- aarch64-unknown-linux-gnu + +This might be the go-to way, if you want to use CLEO in your pipeline. +Once downloaded, extract the files with the command `tar xvf opendut-cleo-{architecture}.tar.gz`. It will then be extracted into +the folder which is the current work directory. You might want to use another directory of your choice. +The tarball contains the `set-env-var.sh` shell script. It can be executed by the command `source set-env-var.sh`, which then sets the +following environment variables to run CLEO: +```` +OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SCOPES +OPENDUT_CLEO_NETWORK_TLS_DOMAIN_NAME_OVERRIDE +OPENDUT_CLEO_NETWORK_TLS_CA +OPENDUT_CLEO_NETWORK_CARL_HOST +OPENDUT_CLEO_NETWORK_CARL_PORT +OPENDUT_CLEO_NETWORK_OIDC_ENABLED +OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ISSUER_URL +SSL_CERT_FILE +```` + +`SSL_CERT_FILE` is a mandatory environment variable for the current state of the implementation and has the same value as the +`OPENDUT_CLEO_NETWORK_TLS_CA`. This might change in the future. + +The script will not set the environment variables for CLIENT_ID and CLIENT_SECRET. This has to be done by the users themselves. +This can easily be done by entering the following commands: +```` +export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ID={{ CLIENT ID VARIABLE }} +export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SECRET={{ CLIENT SECRET VARIABLE }} +```` +These two variables can be obtained by logging in to Keycloak. + +### TL;DR +1. Download archive from `https://{CARL-HOST}/api/cleo/{architecture}/download` +2. Extract `tar xvf opendut-cleo-{architecture}.tar.gz` +3. Execute `source set-env-var.sh` +4. Add two environment variable `export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ID={{ CLIENT ID VARIABLE }}` and `export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SECRET={{ CLIENT SECRET VARIABLE }}` + ## Additional notes - The CA certificate to be provided for CLEO depends on the used certificate authority used on server side for CARL. diff --git a/opendut-carl/Cargo.toml b/opendut-carl/Cargo.toml index a95643c3c..1302d3b04 100644 --- a/opendut-carl/Cargo.toml +++ b/opendut-carl/Cargo.toml @@ -18,6 +18,7 @@ axum = { workspace = true } axum-server = { workspace = true, features = ["tls-rustls"] } axum-server-dual-protocol = { workspace = true } config = { workspace = true } +flate2 = { workspace = true } futures = { workspace = true } googletest = { workspace = true } http = { workspace = true } @@ -29,6 +30,7 @@ opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } serde = { workspace = true, features = ["derive"] } shadow-rs = { workspace = true, default-features = true } +tar = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } tokio-stream = { workspace = true, features = ["full"] } @@ -42,6 +44,7 @@ url = { workspace = true, features = ["serde"] } uuid = { workspace = true } [dev-dependencies] +assert_fs = { workspace = true } async-trait = { workspace = true } rstest = { workspace = true } uuid = { workspace = true, features = ["v4"] } diff --git a/opendut-carl/src/handler/cleo.rs b/opendut-carl/src/handler/cleo.rs new file mode 100644 index 000000000..c871eb145 --- /dev/null +++ b/opendut-carl/src/handler/cleo.rs @@ -0,0 +1,79 @@ +use axum::body::StreamBody; +use axum::extract::{Path, State}; +use axum::response::IntoResponse; +use axum_server_dual_protocol::tokio_util::io::ReaderStream; +use http::{header, StatusCode}; +use opendut_util::project; +use crate::{CleoInstallPath}; + +use crate::util::{CLEO_TARGET_DIRECTORY, CleoArch}; + +pub async fn download_cleo( + Path(architecture): Path, + State(cleo_install_path): State, +) -> impl IntoResponse { + let mut file_name = architecture.file_name(); + let mut content_type = "application/gzip"; + + let cleo_dir = if project::is_running_in_development() { + if file_name != CleoArch::Development.file_name() { + return StatusCode::NOT_FOUND.into_response(); + } + content_type = "application/octet-stream"; + cleo_install_path.0.join("target/debug/opendut-cleo") + } else { + file_name = format!("{}.tar.gz", &file_name); + cleo_install_path.0.join(CLEO_TARGET_DIRECTORY).join(&file_name) + }; + println!("Cleo install directory: {:?}", cleo_dir); + + let file = match tokio::fs::File::open(cleo_dir).await { + Ok(file) => { file } + Err(_) => { return StatusCode::NOT_FOUND.into_response(); } + }; + + let stream = ReaderStream::new(file); + let body = StreamBody::new(stream); + let content_disposition = format!("attachment; filename=\"{}\"", file_name); + let headers = [ + (header::CONTENT_TYPE, content_type), + ( + header::CONTENT_DISPOSITION, + content_disposition.as_str(), + ), + ]; + (headers, body).into_response() +} + +#[cfg(test)] +mod test { + use assert_fs::fixture::{FileTouch, PathChild}; + use assert_fs::TempDir; + use axum::extract::{Path, State}; + use axum::response::IntoResponse; + use googletest::assert_that; + use googletest::matchers::eq; + use http::header; + use crate::CleoInstallPath; + + use crate::util::{CleoArch}; + use crate::handler::cleo::download_cleo; + + #[tokio::test()] + async fn download_cleo_development_succeeds() -> anyhow::Result<()> { + let temp = TempDir::new().unwrap(); + + let dir = temp.child("target/debug/opendut-cleo"); + dir.touch().unwrap(); + + let cleo_install_path = temp.to_path_buf(); + let cleo_state = State::(CleoInstallPath(cleo_install_path)); + let cleo = download_cleo(Path(CleoArch::Development), cleo_state).await; + let response = cleo.into_response(); + let header = response.headers().get(header::CONTENT_DISPOSITION).unwrap(); + let expected_header = format!("attachment; filename=\"{}\"", CleoArch::Development.file_name()); + assert_that!(header.clone().to_str().unwrap(), eq(expected_header.as_str())); + + Ok(()) + } +} \ No newline at end of file diff --git a/opendut-carl/src/handler/mod.rs b/opendut-carl/src/handler/mod.rs new file mode 100644 index 000000000..03433d098 --- /dev/null +++ b/opendut-carl/src/handler/mod.rs @@ -0,0 +1 @@ +pub mod cleo; \ No newline at end of file diff --git a/opendut-carl/src/lib.rs b/opendut-carl/src/lib.rs index eb3335871..9c5931d20 100644 --- a/opendut-carl/src/lib.rs +++ b/opendut-carl/src/lib.rs @@ -1,6 +1,7 @@ extern crate core; use std::net::SocketAddr; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; @@ -15,14 +16,14 @@ use futures::future::BoxFuture; use futures::TryFutureExt; use http::{header::CONTENT_TYPE, Request}; use itertools::Itertools; -use pem::Pem; -use serde::Serialize; +use pem::{Pem}; +use serde::{Serialize}; use shadow_rs::formatcp; use tokio::fs; use tonic::transport::Server; use tower::{BoxError, make::Shared, ServiceExt, steer::Steer}; use tower_http::services::{ServeDir, ServeFile}; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use url::Url; use uuid::Uuid; @@ -35,10 +36,12 @@ use crate::cluster::manager::{ClusterManager, ClusterManagerOptions, ClusterMana use crate::grpc::{ClusterManagerFacade, MetadataProviderFacade, PeerManagerFacade, PeerManagerFacadeOptions, PeerMessagingBrokerFacade}; use crate::peer::broker::{PeerMessagingBroker, PeerMessagingBrokerOptions, PeerMessagingBrokerRef}; use crate::peer::oidc_client_manager::{CarlIdentityProviderConfig, OpenIdConnectClientManager}; +use crate::provisioning::cleo::CleoScript; use crate::resources::manager::{ResourcesManager, ResourcesManagerRef}; use crate::vpn::Vpn; pub mod grpc; +pub mod util; opendut_util::app_info!(); @@ -49,6 +52,8 @@ mod peer; mod resources; pub mod settings; mod vpn; +mod handler; +mod provisioning; #[tracing::instrument] pub async fn create_with_logging(settings_override: config::Config) -> Result<()> { @@ -160,7 +165,7 @@ pub async fn create(settings: LoadedConfig) -> Result<()> { //TODO Arc::clone(&resources_manager), vpn, Clone::clone(&carl_url), - ca, + ca.clone(), oidc_client_manager, peer_manager_facade_options ); @@ -197,7 +202,8 @@ pub async fn create(settings: LoadedConfig) -> Result<()> { //TODO lea_config: LeaConfig { carl_url, idp_config: lea_idp_config, - } + }, + cleo_installation_directory: CleoInstallPath(project::make_path_absolute(PathBuf::from(".")).expect("Could not determine installation directory.")) }; let lea_index_html = lea_dir.join("index.html").clone(); @@ -212,6 +218,15 @@ pub async fn create(settings: LoadedConfig) -> Result<()> { //TODO panic!("Failed to check if LEA index.html exists in: {}", lea_index_html.display()); } } + + if !project::is_running_in_development() { + provisioning::cleo::create_cleo_install_script( + ca, + &app_state.cleo_installation_directory.0, + CleoScript::from_setting(&settings).expect("Could not read settings.") + ).expect("Could not create cleo install script."); + } + let http = axum::Router::new() .fallback_service( axum::Router::new() @@ -220,6 +235,7 @@ pub async fn create(settings: LoadedConfig) -> Result<()> { //TODO ServeDir::new(&licenses_dir) .fallback(ServeFile::new(licenses_dir.join("index.json"))) ) + .route("/api/cleo/:architecture/download", get(handler::cleo::download_cleo)) .route("/api/lea/config", get(lea_config)) .nest_service( "/", @@ -271,7 +287,8 @@ pub async fn create(settings: LoadedConfig) -> Result<()> { //TODO #[derive(Clone)] struct AppState { - lea_config: LeaConfig + lea_config: LeaConfig, + cleo_installation_directory: CleoInstallPath, } #[derive(Clone, Debug, Serialize)] @@ -325,3 +342,12 @@ impl FromRef for LeaConfig { async fn lea_config(State(config): State) -> Json { Json(Clone::clone(&config)) } + +#[derive(Clone, Serialize)] +pub struct CleoInstallPath(pub PathBuf); + +impl FromRef for CleoInstallPath { + fn from_ref(app_state: &AppState) -> Self { + Clone::clone(&app_state.cleo_installation_directory) + } +} diff --git a/opendut-carl/src/peer/broker.rs b/opendut-carl/src/peer/broker.rs index da00348ba..b0d0b3791 100644 --- a/opendut-carl/src/peer/broker.rs +++ b/opendut-carl/src/peer/broker.rs @@ -17,6 +17,7 @@ use opendut_carl_api::proto::services::peer_messaging_broker::upstream; use opendut_types::peer::PeerId; use opendut_types::peer::configuration::PeerConfiguration; use opendut_types::peer::state::{PeerState, PeerUpState}; +use opendut_types::ShortName; use crate::resources::manager::ResourcesManagerRef; @@ -94,6 +95,15 @@ impl PeerMessagingBroker { } self.resources_manager.resources_mut(|resources| { + match resources.get::(peer_id) { + None => { + info!("Peer <{}> opened stream which has not been seen before.", peer_id); + } + Some(peer_state) => { + debug!("Peer <{}> opened stream which was previously in state: {}", peer_id, peer_state.short_name()); + } + }; + resources.update::(peer_id) .modify(|peer_state| match peer_state { PeerState::Up { inner: _, remote_host: peer_remote_host } => { @@ -115,7 +125,7 @@ impl PeerMessagingBroker { error!("Failed to send ApplyPeerConfiguration message: {error}") }; } else { - error!("Failed to send ApplyPeerConfiguration message, because no PeerConfiguration found for peer: {peer_id}") + error!("Failed to send ApplyPeerConfiguration message, because no PeerConfiguration found for peer <{peer_id}>.") } let timeout_duration = self.options.peer_disconnect_timeout; diff --git a/opendut-carl/src/provisioning/cleo.rs b/opendut-carl/src/provisioning/cleo.rs new file mode 100644 index 000000000..deb4999c9 --- /dev/null +++ b/opendut-carl/src/provisioning/cleo.rs @@ -0,0 +1,160 @@ +use std::fmt::{Display, Formatter}; +use std::fs::File; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path}; +use config::Config; +use flate2::Compression; +use flate2::write::GzEncoder; +use pem::Pem; +use tar::Header; +use tracing::log::warn; +use crate::util::{CLEO_TARGET_DIRECTORY, CleoArch}; + +const CA_CERTIFICATE_FILE_NAME: &str = "ca.pem"; + +pub struct CleoScript { + carl_host: String, + carl_port: u16, + oidc_enabled: bool, + issuer_url: String, +} + +impl CleoScript { + pub fn from_setting(settings: &Config) -> anyhow::Result { + Ok(Self { + carl_host: settings.get_string("network.remote.host")?, + carl_port: settings.get_int("network.remote.port")? as u16, + oidc_enabled: settings.get_bool("network.oidc.enabled")?, + issuer_url: settings.get_string("network.oidc.client.issuer.url")?, + }) + } +} + +impl Display for CleoScript { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str( + format!(r#"#!/bin/bash + +DIR_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +CERT_PATH=$DIR_PATH/{} + +export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_SCOPES= +export OPENDUT_CLEO_NETWORK_TLS_DOMAIN_NAME_OVERRIDE={} +export OPENDUT_CLEO_NETWORK_TLS_CA=$CERT_PATH +export OPENDUT_CLEO_NETWORK_CARL_HOST={} +export OPENDUT_CLEO_NETWORK_CARL_PORT={} +export OPENDUT_CLEO_NETWORK_OIDC_ENABLED={} +export OPENDUT_CLEO_NETWORK_OIDC_CLIENT_ISSUER_URL={} +export SSL_CERT_FILE=$CERT_PATH"#, + CA_CERTIFICATE_FILE_NAME, + self.carl_host, + self.carl_host, + self.carl_port, + self.oidc_enabled, + self.issuer_url + ).as_str() + ) + } +} + +pub fn create_cleo_install_script( + ca: Pem, + cleo_install_path: &Path, + cleo_script: CleoScript, +) -> anyhow::Result<()> { + const SET_ENVIRONMENT_VARIABLES_SCRIPT_NAME: &str = "set-env-var.sh"; + const PERMISSION_CODE: u32 = 0o775; + + let mut ca_header = Header::new_gnu(); + ca_header.set_size(ca.contents().len() as u64); + ca_header.set_cksum(); + + let script_path = cleo_install_path.join(CLEO_TARGET_DIRECTORY).join(SET_ENVIRONMENT_VARIABLES_SCRIPT_NAME); + + match std::fs::write( + &script_path, + format!("{}", cleo_script) + ) { + Ok(_) => {} + Err(error) => { warn!("Could not write {}: {}", SET_ENVIRONMENT_VARIABLES_SCRIPT_NAME, error) } + }; + + let mut permissions = std::fs::metadata(&script_path)?.permissions(); + permissions.set_mode(PERMISSION_CODE); + std::fs::set_permissions(&script_path, permissions)?; + + let script_file = &mut File::open(script_path)?; + + for arch in CleoArch::arch_iterator() { + let cleo_file = cleo_install_path.join(CLEO_TARGET_DIRECTORY).join(&arch.file_name()); + let file_name = format!("{}.tar.gz", &arch.file_name()); + let file_path = cleo_install_path.join(CLEO_TARGET_DIRECTORY).join(&file_name); + + match File::create(file_path) { + Ok(tar_gz) => { + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut tar = tar::Builder::new(enc); + tar.append_file( + &arch.file_name(), + &mut File::open(&cleo_file)? + )?; + tar.append_file(SET_ENVIRONMENT_VARIABLES_SCRIPT_NAME, script_file)?; + tar.append_data(&mut ca_header, CA_CERTIFICATE_FILE_NAME, ca.contents())?; + tar.into_inner()?; + } + Err(_) => { + warn!("Could not create {}.tar.gz", &file_name); + } + }; + } + + Ok(()) +} + +#[cfg(test)] +mod test { + use std::path::{PathBuf}; + use std::str::FromStr; + use assert_fs::fixture::{FileTouch, PathChild}; + use assert_fs::TempDir; + use googletest::assert_that; + use googletest::prelude::eq; + use pem::Pem; + use crate::provisioning::cleo::{CleoScript, create_cleo_install_script}; + use crate::util::{CLEO_TARGET_DIRECTORY, CleoArch}; + + #[tokio::test()] + async fn creating_cleo_install_script_succeeds() -> anyhow::Result<()> { + + let temp = TempDir::new().unwrap(); + let dir = temp.child(CLEO_TARGET_DIRECTORY); + std::fs::create_dir_all(dir).unwrap(); + + for arch in CleoArch::arch_iterator() { + let file = temp.child(PathBuf::from(CLEO_TARGET_DIRECTORY).join(arch.file_name())); + file.touch().unwrap(); + } + + let cert = match Pem::from_str(include_str!("../../../resources/development/tls/insecure-development-ca.pem")) { + Ok(cert) => { cert } + Err(_) => { panic!("Not a valid certificate!") } + }; + + create_cleo_install_script( + cert, + &temp.to_path_buf(), + CleoScript { + carl_host: "carl".to_string(), + carl_port: 443, + oidc_enabled: true, + issuer_url: "https://keycloak/realms/opendut/".to_string(), + } + )?; + + for arch in CleoArch::arch_iterator() { + assert_that!(temp.join("opendut-cleo").join(format!("{}.tar.gz",arch.file_name())).exists(), eq(true)); + } + + Ok(()) + } +} diff --git a/opendut-carl/src/provisioning/mod.rs b/opendut-carl/src/provisioning/mod.rs new file mode 100644 index 000000000..03433d098 --- /dev/null +++ b/opendut-carl/src/provisioning/mod.rs @@ -0,0 +1 @@ +pub mod cleo; \ No newline at end of file diff --git a/opendut-carl/src/util/mod.rs b/opendut-carl/src/util/mod.rs new file mode 100644 index 000000000..ecebc044d --- /dev/null +++ b/opendut-carl/src/util/mod.rs @@ -0,0 +1,32 @@ +use std::slice::Iter; +use serde::{Deserialize, Serialize}; +use crate::util::CleoArch::{Arm64, Armhf, X86_64, Development}; + +pub const CLEO_TARGET_DIRECTORY: &str = "opendut-cleo"; + +#[derive(Serialize, Deserialize)] +pub enum CleoArch { + #[serde(rename="x86_64-unknown-linux-gnu")] + X86_64, + #[serde(rename="armv7-unknown-linux-gnueabihf")] + Armhf, + #[serde(rename="aarch64-unknown-linux-gnu")] + Arm64, + #[serde(rename="development")] + Development, +} +impl CleoArch { + pub fn file_name(&self) -> String { + match self { + X86_64 => "opendut-cleo-x86_64-unknown-linux-gnu", + Armhf => "opendut-cleo-armv7-unknown-linux-gnueabihf", + Arm64 => "opendut-cleo-aarch64-unknown-linux-gnu", + Development => "opendut-cleo", + }.to_string() + } + + pub fn arch_iterator() -> Iter<'static, CleoArch> { + static CLEO_ARCH: [CleoArch; 3] = [X86_64, Armhf, Arm64]; + CLEO_ARCH.iter() + } +} \ No newline at end of file