From cb582bc103ddfcb3b3953f287d9d001562852204 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Fri, 13 Dec 2024 18:47:18 -0300 Subject: [PATCH 1/4] feat: remove workers from the proxy configuration file --- CHANGELOG.md | 1 + bin/tx-prover/README.md | 22 +++-------- bin/tx-prover/src/commands/mod.rs | 59 +---------------------------- bin/tx-prover/src/commands/proxy.rs | 19 +++++++--- bin/tx-prover/src/proxy/mod.rs | 21 ---------- bin/tx-prover/src/proxy/worker.rs | 13 +------ 6 files changed, 21 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5d8cd4e3..53df31655 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changes +- Removed workers list from the proxy configuration file (#1018). - Added health check endpoints to the prover service (#1006). - Implemented serialization for `AccountHeader` (#996). - Updated Pingora crates to 0.4 and added polling time to the configuration file (#997). diff --git a/bin/tx-prover/README.md b/bin/tx-prover/README.md index ac3282484..b05a5d646 100644 --- a/bin/tx-prover/README.md +++ b/bin/tx-prover/README.md @@ -42,7 +42,7 @@ First, you need to create a configuration file for the proxy with: miden-tx-prover init ``` -This will create the `miden-tx-prover.toml` file in your current directory. This file will hold the configuration for the proxy. You can modify the configuration by changing the host and ports of the services, and add workers. An example of a valid configuration is: +This will create the `miden-tx-prover.toml` file in your current directory. This file will hold the configuration for the proxy. You can modify the configuration by changing the host and ports of the services, the maximum size of the queue, among other options. An example configuration is: ```toml # Host of the proxy server @@ -61,27 +61,17 @@ max_retries_per_request = 1 max_req_per_sec = 5 # Interval to check the health of the workers health_check_interval_secs = 1 - -[[workers]] -host = "0.0.0.0" -port = 8083 - -[[workers]] -host = "0.0.0.0" -port = 8084 ``` -To add more workers, you will need to add more items with the `[[workers]]` tags. - Then, to start the proxy service, you will need to run: ```bash -miden-tx-prover start-proxy +miden-tx-prover start-proxy [worker1] [worker2] ... [workerN] ``` -This command will start the proxy using the workers defined in the configuration file to send transaction witness to prove. +This command will start the proxy using the workers passed as arguments. The workers should be in the format `host:port`. If no workers are passed, the proxy will start without any workers and will not be able to handle any requests. -At the moment, when a worker added to the proxy stops working and can not connect to it for a request, the connection is marked as retriable meaning that the proxy will try reaching the following worker in a round-robin fashion. The amount of retries is configurable changing the `max_retries_per_request` value in the configuration file. +At the moment, when a worker added to the proxy stops working and can not connect to it for a request, the connection is marked as retriable meaning that the proxy will try reaching another worker. The amount of retries is configurable changing the `max_retries_per_request` value in the configuration file. ## Updating workers on a running proxy @@ -100,15 +90,13 @@ miden-tx-prover add-workers 0.0.0.0:8085 200.58.70.4:50051 miden-tx-prover remove-workers 158.12.12.3:8080 122.122.6.6:50051 ``` -This changes will be persisted to the configuration file. - Note that, in order to update the workers, the proxy must be running in the same computer as the command is being executed because it will check if the client address is localhost to avoid any security issues. ### Health check The worker service implements the [gRPC Health Check](https://grpc.io/docs/guides/health-checking/) standard, and includes the methods described in this [official proto file](https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto). -The proxy service uses this health check to determine if a worker is available to receive requests. If a worker is not available, it will be removed from the set of workers that the proxy can use to send requests, and will persist this change in the configuration file. +The proxy service uses this health check to determine if a worker is available to receive requests. If a worker is not available, it will be removed from the set of workers that the proxy can use to send requests. ## Logging diff --git a/bin/tx-prover/src/commands/mod.rs b/bin/tx-prover/src/commands/mod.rs index 746dd6c96..81cd6015e 100644 --- a/bin/tx-prover/src/commands/mod.rs +++ b/bin/tx-prover/src/commands/mod.rs @@ -1,5 +1,3 @@ -use std::{fs::File, io::Write}; - use clap::Parser; use figment::{ providers::{Format, Toml}, @@ -9,7 +7,6 @@ use init::Init; use miden_tx_prover::PROVER_SERVICE_CONFIG_FILE_NAME; use proxy::StartProxy; use serde::{Deserialize, Serialize}; -use tracing::debug; use update_workers::{AddWorkers, RemoveWorkers, UpdateWorkers}; use worker::StartWorker; @@ -24,8 +21,6 @@ pub mod worker; /// It allows manual modification of the configuration file. #[derive(Serialize, Deserialize)] pub struct ProxyConfig { - /// List of workers used by the proxy. - pub workers: Vec, /// Host of the proxy. pub host: String, /// Port of the proxy. @@ -49,7 +44,6 @@ pub struct ProxyConfig { impl Default for ProxyConfig { fn default() -> Self { Self { - workers: vec![WorkerConfig::new("0.0.0.0", 8083), WorkerConfig::new("0.0.0.0", 8084)], host: "0.0.0.0".into(), port: 8082, timeout_secs: 100, @@ -77,55 +71,6 @@ impl ProxyConfig { .extract() .map_err(|err| format!("Failed to load {} config file: {err}", config_path.display())) } - - /// Saves the configuration to the config file - /// - /// This method will serialize the configuration to a TOML string and write it to the file with - /// the name defined at the [PROVER_SERVICE_CONFIG_FILE_NAME] constant in the current directory. - pub(crate) fn save_to_config_file(&self) -> Result<(), String> { - let mut current_dir = std::env::current_dir().map_err(|err| err.to_string())?; - current_dir.push(PROVER_SERVICE_CONFIG_FILE_NAME); - let config_path = current_dir.as_path(); - - let config_as_toml_string = toml::to_string_pretty(self) - .map_err(|err| format!("error formatting config: {err}"))?; - - let mut file_handle = File::options() - .write(true) - .truncate(true) - .open(config_path) - .map_err(|err| format!("error opening the file: {err}"))?; - - file_handle - .write(config_as_toml_string.as_bytes()) - .map_err(|err| format!("error writing to file: {err}"))?; - - debug!("Config updated successfully"); - - Ok(()) - } - - /// Updates the workers in the configuration with the new list. - pub(crate) fn set_workers(workers: Vec) -> Result<(), String> { - let mut proxy_config = Self::load_config_from_file()?; - - proxy_config.workers = workers; - - proxy_config.save_to_config_file() - } -} - -/// Configuration for a worker -#[derive(Serialize, Deserialize)] -pub struct WorkerConfig { - pub host: String, - pub port: u16, -} - -impl WorkerConfig { - pub fn new(host: &str, port: u16) -> Self { - Self { host: host.into(), port } - } } /// Root CLI struct @@ -156,13 +101,11 @@ pub enum Command { StartProxy(StartProxy), /// Adds workers to the proxy. /// - /// This method will make a request to the proxy defined in the config file to add workers. It - /// will update the configuration file with the new list of workers. + /// This method will make a request to the proxy defined in the config file to add workers. AddWorkers(AddWorkers), /// Removes workers from the proxy. /// /// This method will make a request to the proxy defined in the config file to remove workers. - /// It will update the configuration file with the new list of workers. RemoveWorkers(RemoveWorkers), } diff --git a/bin/tx-prover/src/commands/proxy.rs b/bin/tx-prover/src/commands/proxy.rs index f18c5d998..2f1fb920f 100644 --- a/bin/tx-prover/src/commands/proxy.rs +++ b/bin/tx-prover/src/commands/proxy.rs @@ -10,25 +10,32 @@ use pingora_proxy::http_proxy_service; use crate::proxy::{LoadBalancer, LoadBalancerState}; /// Starts the proxy defined in the config file. +/// +/// Example: `miden-tx-prover start-proxy 0.0.0.0:8080 127.0.0.1:9090` #[derive(Debug, Parser)] -pub struct StartProxy; +pub struct StartProxy { + /// List of workers as host:port strings. + /// + /// Example: `127.0.0.1:8080 192.168.1.1:9090` + #[clap(value_name = "WORKERS")] + workers: Vec, +} impl StartProxy { /// Starts the proxy defined in the config file. /// - /// This method will first read the config file to get the list of workers to start. It will - /// then start a proxy with each worker as a backend. + /// This method will first read the config file to get the parameters for the proxy. It will + /// then start a proxy with each worker passed as command argument as a backend. pub async fn execute(&self) -> Result<(), String> { let mut server = Server::new(Some(Opt::default())).map_err(|err| err.to_string())?; server.bootstrap(); let proxy_config = super::ProxyConfig::load_config_from_file()?; - let workers = proxy_config + let workers = self .workers .iter() - .map(|worker| format!("{}:{}", worker.host, worker.port)) - .map(|worker| Backend::new(&worker).map_err(|err| err.to_string())) + .map(|worker| Backend::new(worker).map_err(|err| err.to_string())) .collect::, String>>()?; let worker_lb = LoadBalancerState::new(workers, &proxy_config).await?; diff --git a/bin/tx-prover/src/proxy/mod.rs b/bin/tx-prover/src/proxy/mod.rs index fd18188d5..0ceacb1ca 100644 --- a/bin/tx-prover/src/proxy/mod.rs +++ b/bin/tx-prover/src/proxy/mod.rs @@ -110,12 +110,8 @@ impl LoadBalancerState { /// - If the worker exists in the current workers list, remove it. /// - Otherwise, do nothing. /// - /// Finally, updates the configuration file with the new list of workers. - /// /// # Errors /// - If the worker cannot be created. - /// - If the configuration cannot be loaded. - /// - If the configuration cannot be saved. pub async fn update_workers( &self, update_workers: UpdateWorkers, @@ -152,11 +148,6 @@ impl LoadBalancerState { }, } - let new_list_of_workers = - workers.iter().map(|worker| worker.try_into()).collect::, _>>()?; - - ProxyConfig::set_workers(new_list_of_workers)?; - info!("Workers updated: {:?}", workers); Ok(()) @@ -556,7 +547,6 @@ impl BackgroundService for LoadBalancerState { /// /// # Errors /// - If the worker has an invalid URI. - /// - If a [WorkerConfig] cannot be created from a given [Worker]. fn start<'life0, 'async_trait>( &'life0 self, _shutdown: ShutdownWatch, @@ -575,17 +565,6 @@ impl BackgroundService for LoadBalancerState { // Update the worker list with healthy workers *workers = healthy_workers; - // Persist the updated worker list to the configuration file - let worker_configs = workers - .iter() - .map(|worker| worker.try_into()) - .collect::, _>>() - .expect("Failed to convert workers to worker configs"); - - if let Err(err) = ProxyConfig::set_workers(worker_configs) { - error!("Failed to update workers in the configuration file: {}", err); - } - // Sleep for the defined interval before the next health check sleep(self.health_check_frequency).await; } diff --git a/bin/tx-prover/src/proxy/worker.rs b/bin/tx-prover/src/proxy/worker.rs index 47a9c7b24..f42466df4 100644 --- a/bin/tx-prover/src/proxy/worker.rs +++ b/bin/tx-prover/src/proxy/worker.rs @@ -7,7 +7,7 @@ use tonic_health::pb::{ }; use tracing::error; -use crate::{commands::WorkerConfig, utils::create_health_check_client}; +use crate::utils::create_health_check_client; // WORKER // ================================================================================================ @@ -72,14 +72,3 @@ impl PartialEq for Worker { self.backend == other.backend } } - -impl TryInto for &Worker { - type Error = String; - - fn try_into(self) -> std::result::Result { - self.backend - .as_inet() - .ok_or_else(|| "Failed to get worker address".to_string()) - .map(|worker_addr| WorkerConfig::new(&worker_addr.ip().to_string(), worker_addr.port())) - } -} From 034dfa778c94f4497faebd14cbd6a875bb09b72f Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Mon, 16 Dec 2024 10:51:13 -0300 Subject: [PATCH 2/4] review: add warning when initializing the proxy w/o workers --- bin/tx-prover/src/commands/proxy.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/tx-prover/src/commands/proxy.rs b/bin/tx-prover/src/commands/proxy.rs index 2f1fb920f..1eb693cfa 100644 --- a/bin/tx-prover/src/commands/proxy.rs +++ b/bin/tx-prover/src/commands/proxy.rs @@ -6,6 +6,7 @@ use pingora::{ server::Server, }; use pingora_proxy::http_proxy_service; +use tracing::warn; use crate::proxy::{LoadBalancer, LoadBalancerState}; @@ -38,6 +39,10 @@ impl StartProxy { .map(|worker| Backend::new(worker).map_err(|err| err.to_string())) .collect::, String>>()?; + if workers.is_empty() { + warn!("Starting the proxy without any workers"); + } + let worker_lb = LoadBalancerState::new(workers, &proxy_config).await?; let health_check_service = background_service("health_check", worker_lb); From e125e2b0d83b216ad94a219c0f0e1c8293d479a0 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Mon, 16 Dec 2024 11:13:25 -0300 Subject: [PATCH 3/4] review: update retries documentation --- bin/tx-prover/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tx-prover/README.md b/bin/tx-prover/README.md index b05a5d646..cc549b22b 100644 --- a/bin/tx-prover/README.md +++ b/bin/tx-prover/README.md @@ -71,7 +71,7 @@ miden-tx-prover start-proxy [worker1] [worker2] ... [workerN] This command will start the proxy using the workers passed as arguments. The workers should be in the format `host:port`. If no workers are passed, the proxy will start without any workers and will not be able to handle any requests. -At the moment, when a worker added to the proxy stops working and can not connect to it for a request, the connection is marked as retriable meaning that the proxy will try reaching another worker. The amount of retries is configurable changing the `max_retries_per_request` value in the configuration file. +At the moment, when a worker added to the proxy stops working and can not connect to it for a request, the connection is marked as retriable meaning that the proxy will try reaching another worker. The number of retries is configurable via the `max_retries_per_request` value in the configuration file. ## Updating workers on a running proxy From 25f1752ef653f7a633acb5c866db1d3859dd2a4f Mon Sep 17 00:00:00 2001 From: Santiago Pittella <87827390+SantiagoPittella@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:05:37 -0300 Subject: [PATCH 4/4] review: improve documentation about starting proxy w/o workers Co-authored-by: Tomas Rodriguez Dala <43424983+tomyrd@users.noreply.github.com> --- bin/tx-prover/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tx-prover/README.md b/bin/tx-prover/README.md index cc549b22b..080d19d89 100644 --- a/bin/tx-prover/README.md +++ b/bin/tx-prover/README.md @@ -69,7 +69,7 @@ Then, to start the proxy service, you will need to run: miden-tx-prover start-proxy [worker1] [worker2] ... [workerN] ``` -This command will start the proxy using the workers passed as arguments. The workers should be in the format `host:port`. If no workers are passed, the proxy will start without any workers and will not be able to handle any requests. +This command will start the proxy using the workers passed as arguments. The workers should be in the format `host:port`. If no workers are passed, the proxy will start without any workers and will not be able to handle any requests until one is added through the `miden-tx-prover add-worker` command. At the moment, when a worker added to the proxy stops working and can not connect to it for a request, the connection is marked as retriable meaning that the proxy will try reaching another worker. The number of retries is configurable via the `max_retries_per_request` value in the configuration file.