Add backup GraphQL functions

- Added backup create/list GraphQL query function for rocks and postgres.

CHANGELOG.md

@@ -5,6 +5,41 @@ file is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and
 this project adheres to [Semantic
 Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- Added two new GraphQL API methods for backup:
+  - `backup_from_directory`:  This new function uses check points in rocksdb to create a
+    backup and returns the backup filename. The current form of backup is unstable, so we
+    will change to using the share_table_files option of the backup engine in the future
+    when it is supported.
+  - `backup_list`: This new function returns a list of backups.
+
+## [0.16.0] - 2024-01-15
+
+### Added
+
+- Added `ranked_outlier_stream` Graphql API to fetch `RankedOutlier` periodically.
+  - Gets the id of the currently stored `Model`.
+  - Generate a `RankedOutlier` iterator corresponding to the prefix of the
+    `Model`'s id. If not first fetch, generate iterator since the last fetched key.
+  - Stream through the `RankedOutlier` iterator, and repeat the behavior after a
+    period of time.
+
+### Changed
+
+- Changed `Node` fields.
+- Updated review-database to 0.22.1.
+- Updated `column_statistics` according to review-database 0.21.0
+  - Removed `event_range` argument.
+  - Changed the `time` argument to `Vec<NaiveDateTime>`.
+  - After adjustment, `column_statistics` now returns all column statistics of the
+    specified `cluster` and created at the batch timestamp listed in the `time` argument.
+  - The timestamp is now added to the return value field `batch_ts`, representing
+    the batch timestamp for the specified `Statistics`.
+  - The returned `Statistics` are now sorted according to `batch_ts` and `column_index`.
+
 ## [0.15.0] - 2023-11-15
 
 ### Changed

Cargo.toml

@@ -22,10 +22,11 @@ jsonwebtoken = "9"
 lazy_static = "1"
 num-traits = "0.2"
 oinq = { git = "https://github.com/petabi/oinq.git", tag = "0.9.1" }
+regex = "1"
 reqwest = { version = "0.11", default-features = false, features = [
   "rustls-tls-native-roots",
 ] }
-review-database = { git = "https://github.com/petabi/review-database.git", tag = "0.20.0" }
+review-database = { path = "../../review-database" }
 roxy = { git = "https://github.com/aicers/roxy.git", tag = "0.2.1" }
 rustls = "0.21"
 rustls-native-certs = "0.6"

examples/minireview.rs

@@ -6,7 +6,7 @@ use futures::{
     pin_mut,
 };
 use ipnet::IpNet;
-use review_database::{migrate_data_dir, Database, Store};
+use review_database::{backup::BackupConfig, migrate_data_dir, Database, Store};
 use review_web::{self as web, graphql::AgentManager, CertManager};
 use serde::Deserialize;
 use std::{
@@ -106,6 +106,7 @@ impl AgentManager for Manager {
 const DEFAULT_DATABASE_URL: &str = "postgres://review@localhost/review";
 const DEFAULT_SERVER: &str = "localhost";
 const DEFAULT_LOG_PATH: &str = "/data/logs/apps";
+const DEFAULT_ENV_PATH: &str = "/usr/bin:/bin";
 
 pub struct Config {
     data_dir: PathBuf,
@@ -118,6 +119,9 @@ pub struct Config {
     key: PathBuf,
     ca_certs: Vec<PathBuf>,
     ip2location: Option<PathBuf>,
+    database_dir: PathBuf,
+    database_container: String,
+    env_path: String,
     reverse_proxies: Vec<review_web::archive::Config>,
 }
 
@@ -133,6 +137,9 @@ struct ConfigParser {
     key: PathBuf,
     ca_certs: Option<Vec<PathBuf>>,
     ip2location: Option<PathBuf>,
+    database_dir: PathBuf,
+    database_container: String,
+    env_path: String,
     archive: Option<review_web::archive::Config>,
     reverse_proxies: Option<Vec<review_web::archive::Config>>,
 }
@@ -155,7 +162,13 @@ impl Config {
             .set_default("log_dir", DEFAULT_LOG_PATH)
             .context("cannot set the default log path")?
             .set_default("htdocs_dir", env::current_dir()?.join("htdocs").to_str())
-            .context("cannot set the default web directory")?;
+            .context("cannot set the default web directory")?
+            .set_default("database_dir", env::current_dir()?.join("AICE_DB").to_str())
+            .context("cannot set the default database directory")?
+            .set_default("database_container", "aice_db")
+            .context("cannot set the default database container")?
+            .set_default("env_path", DEFAULT_ENV_PATH)
+            .context("cannot set the default environment variable path.")?;
         let config: ConfigParser = if let Some(path) = path {
             builder.add_source(File::with_name(path))
         } else {
@@ -187,6 +200,9 @@ impl Config {
             key: config.key,
             ca_certs: config.ca_certs.unwrap_or_default(),
             ip2location: config.ip2location,
+            database_dir: config.database_dir,
+            database_container: config.database_container,
+            env_path: config.env_path,
             reverse_proxies,
         })
     }
@@ -216,6 +232,11 @@ impl Config {
         &self.database_url
     }
 
+    #[must_use]
+    pub fn env_path(&self) -> &str {
+        &self.env_path
+    }
+
     #[must_use]
     pub fn graphql_srv_addr(&self) -> SocketAddr {
         self.graphql_srv_addr
@@ -234,6 +255,16 @@ impl Config {
         self.ip2location.as_deref()
     }
 
+    #[must_use]
+    pub fn database_dir(&self) -> &Path {
+        self.database_dir.as_ref()
+    }
+
+    #[must_use]
+    pub fn database_container(&self) -> &str {
+        &self.database_container
+    }
+
     #[must_use]
     pub(crate) fn reverse_proxies(&self) -> Vec<review_web::archive::Config> {
         self.reverse_proxies.clone()
@@ -287,6 +318,7 @@ async fn run(config: Config) -> Result<Arc<Notify>> {
             .context("failed to connect to database")?;
         store
     };
+
     let agent_manager = Manager {};
     let cert_reload_handle = Arc::new(Notify::new());
 
@@ -302,7 +334,19 @@ async fn run(config: Config) -> Result<Arc<Notify>> {
             .collect(),
         reverse_proxies: config.reverse_proxies(),
     };
-    let web_srv_shutdown_handle = web::serve(web_config, db, store, ip_locator, agent_manager);
+
+    let backup_cfg = BackupConfig::builder()
+        .backup_path(config.backup_dir())
+        .container(config.database_container())
+        .database_dir(config.database_dir())?
+        .database_url(config.database_url())?
+        .env_path(config.env_path())
+        .build();
+
+    let backup_cfg = Arc::new(RwLock::new(backup_cfg));
+
+    let web_srv_shutdown_handle =
+        web::serve(web_config, db, store, ip_locator, agent_manager, backup_cfg);
 
     Ok(web_srv_shutdown_handle)
 }

src/graphql.rs

@@ -50,7 +50,8 @@ use data_encoding::BASE64;
 use ipnet::IpNet;
 use num_traits::ToPrimitive;
 use review_database::{
-    self as database, types::FromKeyValue, Database, Direction, IterableMap, Role, Store,
+    self as database, backup::BackupConfig, types::FromKeyValue, Database, Direction, IterableMap,
+    Role, Store,
 };
 use std::{
     cmp,
@@ -133,6 +134,7 @@ pub(super) fn schema<B>(
     ip_locator: Option<Arc<Mutex<ip2location::DB>>>,
     cert_manager: Arc<dyn CertManager>,
     cert_reload_handle: Arc<Notify>,
+    db_backup_cfg: Arc<RwLock<BackupConfig>>,
 ) -> Schema
 where
     B: AgentManager + 'static,
@@ -147,7 +149,8 @@ where
     .data(store)
     .data(agent_manager)
     .data(cert_manager)
-    .data(cert_reload_handle);
+    .data(cert_reload_handle)
+    .data(db_backup_cfg);
     if let Some(ip_locator) = ip_locator {
         builder = builder.data(ip_locator);
     }
@@ -163,6 +166,7 @@ pub(super) struct Query(
     cluster::ClusterQuery,
     customer::CustomerQuery,
     data_source::DataSourceQuery,
+    db_management::DbManagementQuery,
     event::EventQuery,
     event::EventGroupQuery,
     filter::FilterQuery,