KLine updater fix. Bump version.

helikon-labs · Nov 29, 2024 · 3c9efde · 3c9efde
1 parent ca452a3
commit 3c9efde
Show file tree

Hide file tree

Showing 34 changed files with 210 additions and 1,161 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/_config/base.toml b/_config/base.toml
@@ -157,7 +157,6 @@ user_registration_per_ip_limit_time_window_mins = 10
 user_registration_per_ip_limit = 10
 
 [kline_updater]
-tmp_dir_path = "/path/to/the/temporary/image/dir"
 sleep_seconds = 3600
 begin_year = 2022
 begin_month = 1

diff --git a/_docker/compose/.env.sample b/_docker/compose/.env.sample
@@ -1,4 +1,4 @@
-VERSION=0.22.3
+VERSION=0.22.4
 ENV=production
 LOG_LEVEL=debug
 

diff --git a/subvt-app-service/Cargo.toml b/subvt-app-service/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-app-service"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -21,7 +21,6 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.10"
 subvt-config = { path = "../subvt-config" }
-subvt-logging = { path = "../subvt-logging" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-persistence = { path = "../subvt-persistence" }
 subvt-service-common = { path = "../subvt-service-common" }

diff --git a/subvt-block-processor/Cargo.toml b/subvt-block-processor/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-block-processor"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -9,16 +9,12 @@ anyhow = { workspace = true }
 async-lock = "3.4"
 async-recursion = "1.0"
 async-trait = "0.1"
-chrono = { version = "0.4", default-features = true, features = ["serde"] }
-hex = "0.4"
 lazy_static = { workspace = true }
 log = { workspace = true }
 once_cell = "1"
-parity-scale-codec = "3.6"
 rustc-hash = "2.0"
 serde_json = "1.0"
 subvt-config = { path = "../subvt-config" }
-subvt-logging = { path = "../subvt-logging" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-persistence = { path = "../subvt-persistence" }
 subvt-service-common = { path = "../subvt-service-common" }

diff --git a/subvt-config/Cargo.toml b/subvt-config/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-config"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 

diff --git a/subvt-config/src/lib.rs b/subvt-config/src/lib.rs
@@ -268,7 +268,6 @@ pub struct ReferendumUpdaterConfig {
 
 #[derive(Clone, Debug, Deserialize)]
 pub struct KLineUpdaterConfig {
-    pub tmp_dir_path: String,
     pub sleep_seconds: u64,
     pub begin_year: i32,
     pub begin_month: u32,

diff --git a/subvt-governance/Cargo.toml b/subvt-governance/Cargo.toml
@@ -1,15 +1,13 @@
 [package]
 name = "subvt-governance"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
 [dependencies]
 anyhow = { workspace = true }
 lazy_static = { workspace = true }
-log = { workspace = true }
 reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "brotli"] }
-serde_json = "1.0"
 subvt-config = { path = "../subvt-config" }
 subvt-types = { path = "../subvt-types" }
 tokio = { version = "1.41", features = ["full"] }
diff --git a/subvt-kline-updater/Cargo.toml b/subvt-kline-updater/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-kline-updater"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -12,13 +12,9 @@ lazy_static = { workspace = true }
 log = { workspace = true }
 once_cell = "1"
 reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "brotli"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
 subvt-config = { path = "../subvt-config" }
-subvt-logging = { path = "../subvt-logging" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-persistence = { path = "../subvt-persistence" }
 subvt-service-common = { path = "../subvt-service-common" }
 subvt-types = { path = "../subvt-types" }
 tokio = { version = "1.41", features = ["full"] }
-zip-extract = "0.2.1"
diff --git a/subvt-kline-updater/src/lib.rs b/subvt-kline-updater/src/lib.rs
@@ -14,10 +14,38 @@ lazy_static! {
     static ref CONFIG: Config = Config::default();
 }
 
-#[derive(Default)]
-pub struct KLineUpdater {}
+type KLineRecord = (
+    u64,
+    String,
+    String,
+    String,
+    String,
+    String,
+    u64,
+    String,
+    u32,
+    String,
+    String,
+    String,
+);
 
-impl KLineUpdater {}
+pub struct KLineUpdater {
+    http_client: reqwest::Client,
+}
+
+impl Default for KLineUpdater {
+    fn default() -> Self {
+        let http_client: reqwest::Client = reqwest::Client::builder()
+            .gzip(true)
+            .brotli(true)
+            .timeout(std::time::Duration::from_secs(
+                CONFIG.http.request_timeout_seconds,
+            ))
+            .build()
+            .unwrap();
+        Self { http_client }
+    }
+}
 
 #[async_trait(?Send)]
 impl Service for KLineUpdater {
@@ -47,8 +75,7 @@ impl Service for KLineUpdater {
             .unwrap();
             let mut day = NaiveDate::from_ymd_opt(now.year(), now.month(), now.day()).unwrap();
             loop {
-                // start from 2 days prior to today
-                day = day.checked_sub_days(Days::new(2)).unwrap();
+                day = day.checked_sub_days(Days::new(1)).unwrap();
                 let start_of_day = NaiveDateTime::from(day);
                 let year = day.year();
                 let month = day.month();
@@ -86,66 +113,37 @@ impl Service for KLineUpdater {
                     continue;
                 }
                 let pair = format!("{}{}", CONFIG.substrate.token_ticker, target_ticker);
-                let zip_file_name =
-                    format!("{}-1d-{}-{}-{}.zip", pair, year, month_padded, day_padded,);
-                let csv_file_name =
-                    format!("{}-1d-{}-{}-{}.csv", pair, year, month_padded, day_padded,);
-                let zip_file_local_path =
-                    format!("{}/{}", CONFIG.kline_updater.tmp_dir_path, zip_file_name,);
-                let csv_file_local_path =
-                    format!("{}/{}", CONFIG.kline_updater.tmp_dir_path, csv_file_name,);
-                if !std::path::Path::new(&zip_file_local_path).is_file() {
-                    let url = format!("https://data.binance.vision/data/spot/daily/klines/{pair}/1d/{zip_file_name}");
-                    log::info!("Downloading file {url}.");
-                    let response = reqwest::get(url).await?;
-                    let mut zip_file = std::fs::File::create(&zip_file_local_path)?;
-                    let mut content = std::io::Cursor::new(response.bytes().await?);
-                    std::io::copy(&mut content, &mut zip_file)?;
-                    log::info!("Download complete. Zip file saved at {zip_file_local_path}.");
-                } else {
-                    log::info!("Zip file exists.");
-                }
-                // unzip
-                if !std::path::Path::new(&csv_file_local_path).is_file() {
-                    let target_dir = std::path::PathBuf::from(&CONFIG.kline_updater.tmp_dir_path);
-                    let zip_source = std::fs::read(&zip_file_local_path)?;
-                    zip_extract::extract(std::io::Cursor::new(zip_source), &target_dir, true)?;
-                    log::info!("Zip file extracted.");
-                } else {
-                    log::info!("CSV file exists.");
-                }
-                // read file
-                let csv_content = std::fs::read_to_string(&csv_file_local_path)?;
-                let fields: Vec<&str> = csv_content.split(",").collect();
+
+                // https://api.binance.com/api/v3/klines?symbol=KSMUSDT&interval=1d&limit=1&startTime=1732838400000
+                let url = format!("https://api.binance.com/api/v3/klines?symbol={pair}&interval=1d&limit=1&startTime={timestamp}");
+                let response = self.http_client.get(&url).send().await?;
+                let records: Vec<KLineRecord> = response.json().await?;
+                let fields = records.first().unwrap();
                 // save record
                 let kline = KLine {
                     id: 0,
-                    open_time: fields[0].parse()?,
+                    open_time: fields.0,
                     source_ticker: CONFIG.substrate.token_ticker.clone(),
                     target_ticker: target_ticker.to_string(),
-                    open: BigDecimal::from_str(fields[1])?,
-                    high: BigDecimal::from_str(fields[2])?,
-                    low: BigDecimal::from_str(fields[3])?,
-                    close: BigDecimal::from_str(fields[4])?,
-                    volume: BigDecimal::from_str(fields[5])?,
-                    close_time: fields[6].parse()?,
-                    quote_volume: BigDecimal::from_str(fields[7])?,
-                    count: fields[8].parse()?,
-                    taker_buy_volume: BigDecimal::from_str(fields[9])?,
-                    taker_buy_quote_volume: BigDecimal::from_str(fields[10])?,
+                    open: BigDecimal::from_str(&fields.1)?,
+                    high: BigDecimal::from_str(&fields.2)?,
+                    low: BigDecimal::from_str(&fields.3)?,
+                    close: BigDecimal::from_str(&fields.4)?,
+                    volume: BigDecimal::from_str(&fields.5)?,
+                    close_time: fields.6,
+                    quote_volume: BigDecimal::from_str(&fields.7)?,
+                    count: fields.8,
+                    taker_buy_volume: BigDecimal::from_str(&fields.9)?,
+                    taker_buy_quote_volume: BigDecimal::from_str(&fields.10)?,
                 };
                 postgres.save_kline(&kline).await?;
                 log::info!(
                     "Saved {}-{target_ticker} {day_padded}-{month_padded}-{year}.",
                     CONFIG.substrate.token_ticker,
                 );
-                // delete temp files
-                let _ = std::fs::remove_file(&zip_file_local_path);
-                let _ = std::fs::remove_file(&csv_file_local_path);
             }
             // publish metrics
             metrics::kline_count().set(postgres.get_kline_count().await? as i64);
-            // publish total count
             log::info!(
                 "K-line updater completed. Will sleep for {} seconds",
                 sleep_seconds

diff --git a/subvt-logging/Cargo.toml b/subvt-logging/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-logging"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 

diff --git a/subvt-metrics/Cargo.toml b/subvt-metrics/Cargo.toml
@@ -1,15 +1,13 @@
 [package]
 name = "subvt-metrics"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
 [dependencies]
-env_logger = "0.11"
 log = { workspace = true }
 once_cell = "1"
 prometheus = "0.13"
-subvt-config = { path = "../subvt-config" }
 thiserror = { workspace = true }
 tokio = { version = "1.41", features = ["time", "rt-multi-thread"] }
 warp = { version = "0.3", default-features = false }
diff --git a/subvt-network-status-server/Cargo.toml b/subvt-network-status-server/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-network-status-server"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -10,8 +10,6 @@ rust-version = "1.67.0"
 anyhow = { workspace = true }
 async-trait = "0.1"
 bus = "2.4"
-chrono = "0.4"
-env_logger = "0.11"
 futures-util = "0.3"
 jsonrpsee = { version = "0.24", features = ["full"] }
 lazy_static = { workspace = true }
@@ -24,5 +22,4 @@ subvt-config = { path = "../subvt-config" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-service-common = { path = "../subvt-service-common" }
 subvt-types = { path = "../subvt-types" }
-subvt-logging = { path = "../subvt-logging" }
 tokio = { version = "1.41", features = ["full"] }
diff --git a/subvt-network-status-updater/Cargo.toml b/subvt-network-status-updater/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-network-status-updater"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -15,7 +15,6 @@ redis = { version = "0.27", features = ["tokio-comp"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 subvt-config = { path = "../subvt-config" }
-subvt-logging = { path = "../subvt-logging" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-service-common = { path = "../subvt-service-common" }
 subvt-substrate-client = { path = "../subvt-substrate-client" }

diff --git a/subvt-nft/Cargo.toml b/subvt-nft/Cargo.toml
@@ -1,13 +1,12 @@
 [package]
 name = "subvt-nft"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
 [dependencies]
 anyhow = { workspace = true }
 lazy_static = { workspace = true }
-log = { workspace = true }
 reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "brotli"] }
 subvt-config = { path = "../subvt-config" }
 subvt-types = { path = "../subvt-types" }
diff --git a/subvt-notification-generator/Cargo.toml b/subvt-notification-generator/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-notification-generator"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -21,7 +21,6 @@ serde_json = "1.0"
 subvt-config = { path = "../subvt-config" }
 subvt-service-common = { path = "../subvt-service-common" }
 subvt-types = { path = "../subvt-types" }
-subvt-logging = { path = "../subvt-logging" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-persistence = { path = "../subvt-persistence" }
 subvt-substrate-client = { path = "../subvt-substrate-client" }

diff --git a/subvt-notification-processor/Cargo.toml b/subvt-notification-processor/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-notification-processor"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -18,15 +18,13 @@ itertools = { workspace = true }
 job_scheduler = "1.2"
 lazy_static = { workspace = true }
 lettre = { version = "0.11", default-features = true, features = ["tokio1-native-tls"]}
-lettre_email = "0.9"
 log = { workspace = true }
 once_cell = "1"
 redis = { version = "0.27", features = ["tokio-comp"] }
 rustc-hash = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 subvt-config = { path = "../subvt-config" }
-subvt-logging = { path = "../subvt-logging" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-persistence = { path = "../subvt-persistence" }
 subvt-service-common = { path = "../subvt-service-common" }

diff --git a/subvt-onekv-updater/Cargo.toml b/subvt-onekv-updater/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-onekv-updater"
-version = "0.22.3"
+version = "0.22.4"
 edition = "2021"
 rust-version = "1.67.0"
 
@@ -11,13 +11,10 @@ chrono = { version = "0.4", default-features = true, features = ["serde"] }
 lazy_static = { workspace = true }
 log = { workspace = true }
 once_cell = "1"
-prometheus = { version = "0.13", features = ["process"] }
-rand = "0.8"
 reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "brotli"] }
 subvt-config = { path = "../subvt-config" }
 subvt-metrics = { path = "../subvt-metrics" }
 subvt-persistence = { path = "../subvt-persistence" }
 subvt-service-common = { path = "../subvt-service-common" }
 subvt-types = { path = "../subvt-types" }
-subvt-logging = { path = "../subvt-logging" }
 tokio = { version = "1.41", features = ["full"] }