diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dba39f92..88eed5cc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,21 +6,24 @@ You can use Ctrl+F to search for specific service version ## [Unreleased] -## [1.0.1.44] - 2022-10-21 [HOTFIX] -- Fix some read_to_end related bugs for error empty content -- Fix operation can stuck due to some timeout problem -- Fix status error caused by concurrent sync states and sync packages in ood-daemon service -- Fix Sqlite concurrent write error in some cases - -## [1.0.1.42] - 2022-10-21 -### Add -- Add encryption and decryption to the crypto module of the cyfs stack -### Changes -- Increase the minimum rust version to 1.63 -- Refactor the logic of dsg-services -- Optimize the uninstall logic of DecApp -- Optimize download logic of Service and DecApp to reduce CPU usage -- bdt performance improve -- Improve the stability of master-slave OOD synchronization -### Fix -- Fix the mount path of data directory in DecApp Service sandbox. \ No newline at end of file +## [1.1.1.82] - 2023-04-15 +### Improvements: +Optimize bdt uptime logic when no udp endpoint is available +Protocol stack optimization guessing mime logic +Protocol stack optimization and DecApp related caching logic +Issue #156: AppManager optimizes the uninstall logic of DecApp +Issue #165, #168: Optimization of AppManager's local repo logic +Issue #170: Optimization of AppManager's state repair logic at startup +Issue #196: The -overwrite parameter of ood-installer can control whether to overwrite the original system-config.toml file +Issue #201: Optimize the hash detection logic of the protocol stack chunk reader + +### Fixes: +Issue #157: AppManager may not stop the timeout DecApp installation command correctly under Windows +Fix the service execution problem of AppManager in docker mode. +Fixed a status problem when AppManager reports decapp to ood-daemon. +Fixed some problems related to NDN chunk. +Bdt fixed multiple panic +Issue #183, #186, #187: Fixed multiple panics in the protocol stack +Issue #185: Use cyfs-async-h1 to replace the original async-h1 to avoid panic caused by invalid http header +Issue #198: AppManager does not handle the timeout of install command correctly in docker mode. +Fix: AppManager incorrectly set the App status when it received the Install command, resulting in the inability to retry the installation logic after reboot \ No newline at end of file diff --git a/doc/release-notes/release-notes-1.1.1.82.md b/doc/release-notes/release-notes-1.1.1.82.md new file mode 100644 index 000000000..4cdbb6a8c --- /dev/null +++ b/doc/release-notes/release-notes-1.1.1.82.md @@ -0,0 +1,21 @@ +## [1.1.1.82] - 2023-04-15 +### Improvements: +Optimize bdt uptime logic when no udp endpoint is available +Protocol stack optimization guessing mime logic +Protocol stack optimization and DecApp related caching logic +Issue #156: AppManager optimizes the uninstall logic of DecApp +Issue #165, #168: Optimization of AppManager's local repo logic +Issue #170: Optimization of AppManager's state repair logic at startup +Issue #196: The -overwrite parameter of ood-installer can control whether to overwrite the original system-config.toml file +Issue #201: Optimize the hash detection logic of the protocol stack chunk reader + +### Fixes: +Issue #157: AppManager may not stop the timeout DecApp installation command correctly under Windows +Fix the service execution problem of AppManager in docker mode. +Fixed a status problem when AppManager reports decapp to ood-daemon. +Fixed some problems related to NDN chunk. +Bdt fixed multiple panic +Issue #183, #186, #187: Fixed multiple panics in the protocol stack +Issue #185: Use cyfs-async-h1 to replace the original async-h1 to avoid panic caused by invalid http header +Issue #198: AppManager does not handle the timeout of install command correctly in docker mode. +Fix: AppManager incorrectly set the App status when it received the Install command, resulting in the inability to retry the installation logic after reboot \ No newline at end of file diff --git a/src/Cargo.lock b/src/Cargo.lock index 02fcc5733..9be462fb3 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -171,6 +171,7 @@ dependencies = [ "serde", "serde_json", "surf", + "sysinfo 0.28.4", "toml 0.5.11", "version-compare", "wait-timeout", @@ -646,6 +647,20 @@ dependencies = [ "md5", ] +[[package]] +name = "bdt-tool" +version = "0.1.0" +dependencies = [ + "async-std", + "clap 2.34.0", + "cyfs-base", + "cyfs-bdt", + "cyfs-debug", + "cyfs-util", + "log 0.4.17", + "md5", +] + [[package]] name = "bincode" version = "1.3.3" @@ -1552,6 +1567,7 @@ dependencies = [ "cyfs-tracker-cache", "cyfs-util", "file-rotate", + "futures", "hmac 0.12.1", "http-types", "log 0.4.17", @@ -1565,6 +1581,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.6", + "surf", "tide", "walkdir", "zip", @@ -2425,6 +2442,7 @@ dependencies = [ "cyfs-util", "futures", "hex", + "http-types", "log 0.4.17", "once_cell", "rand 0.8.5", @@ -4791,6 +4809,7 @@ dependencies = [ "async-std", "async-trait", "cyfs-async-h1", + "cyfs-backup", "cyfs-base", "cyfs-debug", "cyfs-lib", @@ -4806,6 +4825,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "surf", "sysinfo 0.26.9", "tide", ] @@ -7105,6 +7125,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "sysinfo" +version = "0.28.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "take_mut" version = "0.2.2" diff --git a/src/Cargo.toml b/src/Cargo.toml index 80cf5629b..9037e62e2 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -54,6 +54,7 @@ members = [ "./tools/cyfs-check", "./tools/sn-updater", "./tools/cyfs-backup-tool", + "./tools/bdt-tool", "./meta/browser-meta-spv", "./meta/cyfs-meta", diff --git a/src/component/cyfs-backup/Cargo.toml b/src/component/cyfs-backup/Cargo.toml index 17f1ebd31..c8bd83e69 100644 --- a/src/component/cyfs-backup/Cargo.toml +++ b/src/component/cyfs-backup/Cargo.toml @@ -45,6 +45,8 @@ hmac = '0.12' base58 = '0.2.0' tide = "0.16" http-types = "2.12" +surf = { version = '2.3', default-features = false, features = ['h1-client-rustls'] } +futures = "0.3" [dev-dependencies] rand = "0.8" \ No newline at end of file diff --git a/src/component/cyfs-backup/src/archive_download/def.rs b/src/component/cyfs-backup/src/archive_download/def.rs new file mode 100644 index 000000000..306272a59 --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/def.rs @@ -0,0 +1,122 @@ +use cyfs_base::*; + +use http_types::Url; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RemoteArchiveUrl { + pub base_url: String, + pub file_name: Option, + pub query_string: Option, +} + +impl RemoteArchiveUrl { + pub fn parse_url(&self) -> BuckyResult { + let url = match &self.file_name { + Some(file_name) => { + format!("{}/{}", self.base_url.trim_end_matches('/'), file_name) + } + None => self.base_url.clone(), + }; + + let mut url = Url::parse(&url).map_err(|e| { + let msg = format!( + "invalid remote archive url format! {}, {}", + self.base_url, e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::InvalidFormat, msg) + })?; + + if let Some(query) = &self.query_string { + url.set_query(Some(query.as_str())); + } + + Ok(url) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum RemoteArchiveInfo { + ZipFile(RemoteArchiveUrl), + Folder(RemoteArchiveUrl), +} + +impl RemoteArchiveInfo { + // Supports URLs in two formats + // {base_url}?{query_string} + // {base_url}/${filename}?{query_string} + pub fn parse(url: &str) -> BuckyResult { + let (base_url, query_string) = match url.split_once("?") { + Some((base_url, query_string)) => (base_url.to_owned(), Some(query_string.to_owned())), + None => (url.to_owned(), None), + }; + + let ret = match base_url.find("${filename}") { + Some(_) => { + let base_url = base_url.replace("${filename}", ""); + + let info = RemoteArchiveUrl { + base_url, + file_name: None, + query_string, + }; + RemoteArchiveInfo::Folder(info) + } + None => { + let info = RemoteArchiveUrl { + base_url, + file_name: None, + query_string, + }; + RemoteArchiveInfo::ZipFile(info) + } + }; + + Ok(ret) + } +} + +#[cfg(test)] +mod test { + use super::RemoteArchiveInfo; + + #[test] + fn test_url() { + let url = "http://127.0.0.1:1234/a/b?token=123456"; + let info = RemoteArchiveInfo::parse(url).unwrap(); + if let RemoteArchiveInfo::ZipFile(info) = info { + assert_eq!(info.base_url, "http://127.0.0.1:1234/a/b"); + assert_eq!(info.query_string.as_deref(), Some("token=123456")); + } else { + unreachable!(); + } + + let url = "http://127.0.0.1:1234/a/b"; + let info = RemoteArchiveInfo::parse(url).unwrap(); + if let RemoteArchiveInfo::ZipFile(info) = info { + assert_eq!(info.base_url, "http://127.0.0.1:1234/a/b"); + assert_eq!(info.query_string, None); + } else { + unreachable!(); + } + + let url = "http://127.0.0.1:1234/a/b/${filename}?token=123456"; + let info = RemoteArchiveInfo::parse(url).unwrap(); + if let RemoteArchiveInfo::Folder(info) = info { + assert_eq!(info.base_url, "http://127.0.0.1:1234/a/b/"); + assert_eq!(info.query_string.as_deref(), Some("token=123456")); + } else { + unreachable!(); + } + + let url = "http://127.0.0.1:1234/a/b/${filename}"; + let info = RemoteArchiveInfo::parse(url).unwrap(); + if let RemoteArchiveInfo::Folder(info) = info { + assert_eq!(info.base_url, "http://127.0.0.1:1234/a/b/"); + assert_eq!(info.query_string, None); + } else { + unreachable!(); + } + } +} \ No newline at end of file diff --git a/src/component/cyfs-backup/src/archive_download/file_downloader.rs b/src/component/cyfs-backup/src/archive_download/file_downloader.rs new file mode 100644 index 000000000..110c72ccf --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/file_downloader.rs @@ -0,0 +1,141 @@ +use cyfs_base::*; + +use async_std::io::ReadExt; +use async_std::{fs::File, io::WriteExt}; +use http_types::Url; +use std::path::PathBuf; +use surf::Client; + +use super::ArchiveProgressHolder; + +pub struct ArchiveFileDownloader { + file: PathBuf, + url: Url, +} + +impl ArchiveFileDownloader { + pub fn new(url: Url, file: PathBuf) -> Self { + Self { url, file } + } + + pub async fn download(&self, progress: &ArchiveProgressHolder) -> BuckyResult<()> { + progress.begin_file(&self.file.as_os_str().to_string_lossy(), 0); + + let ret = self.download_inner(progress).await; + progress.finish_current_file(ret.clone()); + + ret + } + + async fn download_inner(&self, progress: &ArchiveProgressHolder) -> BuckyResult<()> { + // Get a client instance + let mut res = Client::new().get(&self.url).await.map_err(|e| { + let msg = format!( + "get from remote archive url failed! url={}, {}", + self.url, e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::ConnectFailed, msg) + })?; + + if !res.status().is_success() { + let msg = format!( + "get from remote archive url failed! url={}, status={}", + self.url, res.status(), + ); + error!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::Failed, msg)); + } + + let content_length = res.len().ok_or_else(|| { + let msg = format!( + "get content-length header from remote archive response but not found! url={}", + self.url + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::InvalidFormat, msg) + })?; + + progress.reset_current_file_total(content_length as u64); + + info!( + "will download archive file: {} -> {}, len={}bytes", + self.url, + self.file.display(), + content_length + ); + + // Make sure the dir exists + if let Some(dir) = self.file.parent() { + if !dir.is_dir() { + async_std::fs::create_dir_all(&dir).await.map_err(|e| { + let msg = format!( + "create local archive dir failed! dir={}, {}", + dir.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + } + } + + // Create file and writer instance + let mut file = File::create(&self.file).await.map_err(|e| { + let msg = format!( + "create local archive file for write but failed! file={}, {}", + self.file.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + // Stream download the file with progress + let mut body = res.take_body().into_reader(); + + let mut buf = vec![0; 1024 * 64]; + loop { + let len: usize = body.read(&mut buf).await.map_err(|e| { + let msg = format!("read data from remote failed! url={}, {}", self.url, e); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::ConnectionAborted, msg) + })?; + + if len == 0 { + break; + } + + file.write_all(&buf[..len]).await.map_err(|e| { + let msg = format!( + "write buf to local archive file but failed! file={}, {}", + self.file.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + progress.update_current_file_progress(len as u64); + } + + file.flush().await.map_err(|e| { + let msg = format!( + "flush local file error! file={}, {}", + self.file.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + info!( + "download archive file complete: {} -> {}, len={}bytes", + self.url, + self.file.display(), + content_length + ); + + Ok(()) + } +} diff --git a/src/component/cyfs-backup/src/archive_download/folder_downloader.rs b/src/component/cyfs-backup/src/archive_download/folder_downloader.rs new file mode 100644 index 000000000..aa00651cd --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/folder_downloader.rs @@ -0,0 +1,99 @@ +use crate::archive::ObjectArchiveIndexHelper; +use cyfs_backup_lib::*; +use cyfs_base::*; + +use super::file_downloader::ArchiveFileDownloader; +use super::{def::RemoteArchiveUrl, progress::ArchiveProgressHolder}; + +use std::path::PathBuf; + +pub struct ArchiveFolderDownloader { + url_info: RemoteArchiveUrl, + folder: PathBuf, +} + +impl ArchiveFolderDownloader { + pub fn new(url_info: RemoteArchiveUrl, folder: PathBuf) -> Self { + Self { url_info, folder } + } + + pub async fn download(&self, progress: &ArchiveProgressHolder) -> BuckyResult<()> { + info!( + "will download archive index: url={}", + self.url_info.base_url + ); + + let index = self.download_index(progress).await?; + + info!("download archive index complete: {:?}", index); + + // Sum to got total bytes + let mut total = 0; + index.object_files.iter().for_each(|item| { + total += item.file_len; + }); + + index.chunk_files.iter().for_each(|item| { + total += item.file_len; + }); + + info!( + "will download archive data files: url={}, total={}", + self.url_info.base_url, total + ); + + progress.reset_total(total); + + for item in &index.object_files { + self.download_file(&index, item, progress).await?; + } + + for item in &index.chunk_files { + self.download_file(&index, item, progress).await?; + } + + info!("download archive complete! total={}", total); + + Ok(()) + } + + async fn download_index( + &self, + progress: &ArchiveProgressHolder, + ) -> BuckyResult { + let mut url = self.url_info.clone(); + url.file_name = Some("index".to_owned()); + let url = url.parse_url()?; + + let file = self.folder.join("index"); + let downloader = ArchiveFileDownloader::new(url, file); + downloader.download(progress).await?; + + ObjectArchiveIndexHelper::load(&self.folder).await + } + + async fn download_file( + &self, + index: &ObjectArchiveIndex, + item: &ObjectPackFileInfo, + progress: &ArchiveProgressHolder, + ) -> BuckyResult<()> { + let relative_path = match &index.data_folder { + Some(folder_name) => { + format!("{}/{}", folder_name, item.name) + } + None => item.name.clone(), + }; + + let file = self.folder.join(&relative_path); + + let mut url = self.url_info.clone(); + url.file_name = Some(relative_path); + let url = url.parse_url()?; + + let downloader = ArchiveFileDownloader::new(url, file); + downloader.download(progress).await?; + + Ok(()) + } +} diff --git a/src/component/cyfs-backup/src/archive_download/helper.rs b/src/component/cyfs-backup/src/archive_download/helper.rs new file mode 100644 index 000000000..be5d0fccc --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/helper.rs @@ -0,0 +1,39 @@ +use cyfs_base::*; + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + + +#[derive(Clone)] +pub struct TaskAbortHandler { + aborted: Arc +} + +impl TaskAbortHandler { + pub fn new() -> Self { + Self { + aborted: Arc::new(AtomicBool::new(false)) + } + } + + pub fn abort(&self) { + self.aborted.store(true, Ordering::SeqCst); + } + + pub fn is_aborted(&self) -> bool { + self.aborted.load(Ordering::SeqCst) + } + + pub fn check_aborted(&self) -> BuckyResult<()> { + match self.is_aborted() { + true => { + let msg = format!("task already been aborted!"); + warn!("{}", msg); + Err(BuckyError::new(BuckyErrorCode::Aborted, msg)) + } + false => { + Ok(()) + } + } + } +} \ No newline at end of file diff --git a/src/component/cyfs-backup/src/archive_download/mod.rs b/src/component/cyfs-backup/src/archive_download/mod.rs new file mode 100644 index 000000000..24e7a20b8 --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/mod.rs @@ -0,0 +1,13 @@ +mod def; +mod file_downloader; +mod folder_downloader; +mod helper; +mod progress; +mod unzip; + +pub use def::*; +pub use file_downloader::*; +pub use folder_downloader::*; +pub use helper::*; +pub use progress::*; +pub use unzip::*; diff --git a/src/component/cyfs-backup/src/archive_download/progress.rs b/src/component/cyfs-backup/src/archive_download/progress.rs new file mode 100644 index 000000000..8ec7c0445 --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/progress.rs @@ -0,0 +1,120 @@ +use cyfs_base::*; + +use serde::{Deserialize, Serialize}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FileProgress { + pub file: String, + pub total: u64, + pub completed: u64, + pub result: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ArchiveProgress { + pub total: u64, + pub completed: u64, + pub result: Option>, + + pub current: Option, +} + +impl ArchiveProgress { + pub fn new() -> Self { + Self { + total: 0, + completed: 0, + result: None, + current: None, + } + } + + pub fn reset_total(&mut self, total: u64) { + self.total = total; + } + + pub fn finish(&mut self, result: BuckyResult<()>) { + assert!(self.result.is_none()); + self.result = Some(result); + } + + pub fn begin_file(&mut self, file: &str, total: u64) { + // Treat as single file archive mode + if self.total == 0 { + self.total = total; + } + + let file_progress = FileProgress { + file: file.to_owned(), + total, + completed: 0, + result: None, + }; + + self.current = Some(file_progress); + } + + pub fn reset_current_file_total(&mut self, total: u64) { + // Treat as single file archive mode + if self.total == 0 { + self.total = total; + } + + assert!(self.current.is_some()); + self.current.as_mut().unwrap().total = total; + } + + pub fn update_current_file_progress(&mut self, completed: u64) { + assert!(self.current.is_some()); + self.current.as_mut().unwrap().completed = completed; + self.completed += completed; + } + + pub fn finish_current_file(&mut self, result: BuckyResult<()>) { + assert!(self.current.is_some()); + self.current.as_mut().unwrap().result = Some(result); + self.current = None; + } +} + +#[derive(Clone)] +pub struct ArchiveProgressHolder(Arc>); + +impl ArchiveProgressHolder { + pub fn new() -> Self { + Self(Arc::new(Mutex::new(ArchiveProgress::new()))) + } + + pub fn get_progress(&self) -> ArchiveProgress { + self.0.lock().unwrap().clone() + } + + pub fn reset_total(&self, total: u64) { + self.0.lock().unwrap().reset_total(total) + } + + pub fn finish(&self, result: BuckyResult<()>) { + self.0.lock().unwrap().finish(result) + } + + // File progress related methods + pub fn begin_file(&self, file: &str, total: u64) { + self.0.lock().unwrap().begin_file(file, total) + } + + pub fn reset_current_file_total(&self, total: u64) { + self.0.lock().unwrap().reset_current_file_total(total) + } + + pub fn update_current_file_progress(&self, completed: u64) { + self.0 + .lock() + .unwrap() + .update_current_file_progress(completed) + } + + pub fn finish_current_file(&self, result: BuckyResult<()>) { + self.0.lock().unwrap().finish_current_file(result) + } +} diff --git a/src/component/cyfs-backup/src/archive_download/unzip.rs b/src/component/cyfs-backup/src/archive_download/unzip.rs new file mode 100644 index 000000000..5b19f56ad --- /dev/null +++ b/src/component/cyfs-backup/src/archive_download/unzip.rs @@ -0,0 +1,225 @@ +use super::progress::ArchiveProgressHolder; +use cyfs_base::*; +use super::helper::TaskAbortHandler; + +use std::io::prelude::*; +use std::path::{Path, PathBuf}; + + +#[derive(Clone)] +pub struct ArchiveUnzip { + archive_file: PathBuf, + target_folder: PathBuf, +} + +impl ArchiveUnzip { + pub fn new(archive_file: PathBuf, target_folder: PathBuf) -> Self { + Self { + archive_file, + target_folder, + } + } + + pub async fn unzip(&self, progress: &ArchiveProgressHolder, abort_handler: &TaskAbortHandler) -> BuckyResult<()> { + let this = self.clone(); + let progress = progress.clone(); + let abort_handler = abort_handler.clone(); + + let task = async_std::task::spawn_blocking(move || { + this.unzip_inner(&progress, abort_handler) + }); + + task.await + } + + fn unzip_inner(&self, progress: &ArchiveProgressHolder, abort_handler: TaskAbortHandler) -> BuckyResult<()> { + let file = std::fs::File::open(&self.archive_file).map_err(|e| { + let msg = format!( + "open archive file failed! file={}, {}", + self.archive_file.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + // Stat total progress with compressed size + let total = file + .metadata() + .map_err(|e| { + let msg = format!( + "get metadata from archive file failed! file={}, {}", + self.archive_file.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })? + .len(); + + progress.reset_total(total); + + let mut archive = zip::ZipArchive::new(file).map_err(|e| { + let msg = format!( + "open archive file failed! file={}, {}", + self.archive_file.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::InvalidData, msg) + })?; + + abort_handler.check_aborted()?; + + for i in 0..archive.len() { + abort_handler.check_aborted()?; + + let mut file = archive.by_index(i).map_err(|e| { + let msg = format!("load file from archive failed! index={}, {}", i, e); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::InvalidData, msg) + })?; + + #[allow(deprecated)] + let target_file_path = self.target_folder.join(file.sanitized_name()); + + if file.is_dir() { + std::fs::create_dir_all(&target_file_path).map_err(|e| { + let msg = format!( + "create archive inner dir failed! file={}, {}", + target_file_path.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + } else { + #[allow(deprecated)] + let file_path_str = file + .sanitized_name() + .as_os_str() + .to_string_lossy() + .to_string(); + + if file.size() == 0 { + warn!("got zero byte file! {}", file_path_str); + continue; + } + + // Stat current file use compressed size + progress.begin_file(&file_path_str, file.compressed_size()); + + let ret = self + .unzip_file(&mut file, &target_file_path, progress, &abort_handler); + progress.finish_current_file(ret.clone()); + + ret?; + } + } + + Ok(()) + } + + fn unzip_file( + &self, + zip_file: &mut zip::read::ZipFile<'_>, + target_file_path: &Path, + progress: &ArchiveProgressHolder, + abort_handler: &TaskAbortHandler, + ) -> BuckyResult<()> { + if let Some(dir) = target_file_path.parent() { + if !dir.is_dir() { + std::fs::create_dir_all(&dir).map_err(|e| { + let msg = format!( + "create local archive dir failed! dir={}, {}", + dir.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + } + } + + let mut out = std::fs::File::create(&target_file_path).map_err(|e| { + let msg = format!( + "create local archive file failed! dir={}, {}", + target_file_path.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + #[allow(deprecated)] + let file_path_str = zip_file + .sanitized_name() + .as_os_str() + .to_string_lossy() + .to_string(); + + let mut buf = vec![0; 1024 * 64]; + loop { + abort_handler.check_aborted()?; + + let len: usize = zip_file.read(&mut buf).map_err(|e| { + let msg = format!( + "read data from archive failed! file={}, {}", + file_path_str, e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + if len == 0 { + break; + } + + abort_handler.check_aborted()?; + + out.write_all(&buf[..len]).map_err(|e| { + let msg = format!( + "write buf to local archive file failed! file={}, {}", + target_file_path.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + // Estimate a size based on compression ratio + let mut compress_len = len as u64 * zip_file.compressed_size() / zip_file.size(); + if compress_len > zip_file.compressed_size() { + compress_len = zip_file.compressed_size(); + } + progress.update_current_file_progress(compress_len); + } + + abort_handler.check_aborted()?; + + out.flush().map_err(|e| { + let msg = format!( + "flush to local archive file failed! file={}, {}", + target_file_path.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + + progress.update_current_file_progress(zip_file.compressed_size()); + /* + std::io::copy(&mut file, &mut out).map_err(|e| { + let msg = format!( + "extract archive file to local file failed! file={}, {}", + file_path.display(), + e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::IoError, msg) + })?; + */ + + Ok(()) + } +} diff --git a/src/component/cyfs-backup/src/backup/restore.rs b/src/component/cyfs-backup/src/backup/restore.rs index b11c34e1a..9db8dbe1f 100644 --- a/src/component/cyfs-backup/src/backup/restore.rs +++ b/src/component/cyfs-backup/src/backup/restore.rs @@ -5,7 +5,7 @@ use cyfs_base::*; use std::sync::{Arc, Mutex}; pub struct RestoreManager { - // all backup tasks + // all restore tasks tasks: Mutex>, } diff --git a/src/component/cyfs-backup/src/lib.rs b/src/component/cyfs-backup/src/lib.rs index 606e51614..f0cc7ec37 100644 --- a/src/component/cyfs-backup/src/lib.rs +++ b/src/component/cyfs-backup/src/lib.rs @@ -10,10 +10,13 @@ mod restore; mod state_backup; mod uni_backup; mod service; +mod archive_download; +mod remote_restore; pub use backup::*; pub use crypto::*; pub use service::*; +pub use remote_restore::*; #[macro_use] extern crate log; diff --git a/src/component/cyfs-backup/src/remote_restore/def.rs b/src/component/cyfs-backup/src/remote_restore/def.rs new file mode 100644 index 000000000..849a052ce --- /dev/null +++ b/src/component/cyfs-backup/src/remote_restore/def.rs @@ -0,0 +1,30 @@ +use cyfs_backup_lib::*; + +use serde::{Deserialize, Serialize}; + + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RemoteRestoreParams { + // TaskId, should be valid segment string of path + pub id: String, + + // Restore related params + pub cyfs_root: Option, + pub isolate: Option, + pub password: Option, + + // Remote archive info + pub remote_archive: String, +} + +impl RemoteRestoreParams { + pub fn new(id: impl Into, remote_archive: impl Into) -> Self { + Self { + id: id.into(), + cyfs_root: None, + isolate: None, + password: None, + remote_archive: remote_archive.into(), + } + } +} \ No newline at end of file diff --git a/src/component/cyfs-backup/src/remote_restore/manager.rs b/src/component/cyfs-backup/src/remote_restore/manager.rs new file mode 100644 index 000000000..43ef72287 --- /dev/null +++ b/src/component/cyfs-backup/src/remote_restore/manager.rs @@ -0,0 +1,94 @@ +use super::def::*; +use super::status::*; +use super::task::*; +use cyfs_base::*; + +use std::sync::{Arc, Mutex}; + +pub struct RemoteRestoreManager { + // all restore tasks + tasks: Mutex>, +} + +impl RemoteRestoreManager { + pub fn new() -> Self { + Self { + tasks: Mutex::new(vec![]), + } + } + + pub fn get_tasks(&self) -> Vec { + let tasks = self.tasks.lock().unwrap(); + let list: Vec = tasks.iter().map(|task| task.id().to_owned()).collect(); + + list + } + + fn create_remote_restore_task( + &self, + params: &RemoteRestoreParams, + ) -> BuckyResult { + let task = RemoteRestoreTask::new(params.id.clone()); + + { + let mut tasks = self.tasks.lock().unwrap(); + if tasks.iter().find(|item| item.id() == params.id).is_some() { + let msg = format!("remote restore task already exists! task={}", params.id); + error!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::AlreadyExists, msg)); + } + + tasks.push(task.clone()); + } + + Ok(task) + } + + pub async fn run_remote_restore(&self, params: RemoteRestoreParams) -> BuckyResult<()> { + let task = self.create_remote_restore_task(¶ms)?; + + task.run(params).await + } + + pub fn start_remote_restore(&self, params: RemoteRestoreParams) -> BuckyResult<()> { + let mut task = self.create_remote_restore_task(¶ms)?; + + task.start(params) + } + + pub fn get_task_status(&self, id: &str) -> BuckyResult { + let status = { + let tasks = self.tasks.lock().unwrap(); + let ret = tasks.iter().find(|item| item.id() == id); + if ret.is_none() { + let msg = format!("backup task not exists! task={}", id); + error!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::NotFound, msg)); + } + + ret.unwrap().status() + }; + + Ok(status) + } + + pub async fn abort_task(&self, id: &str) -> BuckyResult<()> { + let task = + { + let mut tasks = self.tasks.lock().unwrap(); + if let Some(index) = tasks.iter().position(|task| task.id() == id) { + tasks.swap_remove(index) + } else { + let msg = format!("backup task not exists! task={}", id); + error!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::NotFound, msg)); + } + }; + + task.abort().await; + + Ok(()) + } +} + +pub type RemoteRestoreManagerRef = Arc; diff --git a/src/component/cyfs-backup/src/remote_restore/mod.rs b/src/component/cyfs-backup/src/remote_restore/mod.rs new file mode 100644 index 000000000..8b44c7515 --- /dev/null +++ b/src/component/cyfs-backup/src/remote_restore/mod.rs @@ -0,0 +1,12 @@ +mod def; +mod manager; +mod status; +mod task; + +pub use def::*; +pub use manager::*; +pub use status::*; +pub use task::*; + +#[cfg(test)] +mod test; diff --git a/src/component/cyfs-backup/src/remote_restore/status.rs b/src/component/cyfs-backup/src/remote_restore/status.rs new file mode 100644 index 000000000..a536ce873 --- /dev/null +++ b/src/component/cyfs-backup/src/remote_restore/status.rs @@ -0,0 +1,177 @@ +use crate::archive_download::*; +use crate::backup::RestoreManagerRef; +use cyfs_backup_lib::*; +use cyfs_base::*; + +use serde::{Deserialize, Serialize}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub enum RemoteRestoreTaskPhase { + Init, + Download, + Unpack, + Restore, + Complete, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RemoteRestoreStatus { + pub phase: RemoteRestoreTaskPhase, + pub result: Option>, + + pub download_progress: Option, + pub unpack_progress: Option, + pub restore_status: Option, +} + +impl Default for RemoteRestoreStatus { + fn default() -> Self { + Self { + phase: RemoteRestoreTaskPhase::Init, + result: None, + + download_progress: None, + unpack_progress: None, + restore_status: None, + } + } +} + +pub struct RemoteRestoreStatusManagerInner { + pub phase: RemoteRestoreTaskPhase, + pub result: Option>, + + pub download_progress: Option, + pub unpack_progress: Option, + pub restore_status: Option<(String, RestoreManagerRef)>, +} + +impl RemoteRestoreStatusManagerInner { + pub fn new() -> Self { + Self { + phase: RemoteRestoreTaskPhase::Init, + result: None, + download_progress: None, + unpack_progress: None, + restore_status: None, + } + } + + pub fn begin_download(&mut self, progress: ArchiveProgressHolder) { + assert_eq!(self.phase, RemoteRestoreTaskPhase::Init); + assert!(self.download_progress.is_none()); + + self.phase = RemoteRestoreTaskPhase::Download; + self.download_progress = Some(progress); + } + + pub fn begin_unpack(&mut self, progress: ArchiveProgressHolder) { + assert!(self.unpack_progress.is_none()); + + self.phase = RemoteRestoreTaskPhase::Unpack; + self.unpack_progress = Some(progress); + } + + pub fn begin_restore(&mut self, task_id: &str, restore_manager: RestoreManagerRef) { + assert!(self.restore_status.is_none()); + + self.phase = RemoteRestoreTaskPhase::Restore; + self.restore_status = Some((task_id.to_owned(), restore_manager)); + } + + pub fn complete(&mut self, result: BuckyResult<()>) { + self.phase = RemoteRestoreTaskPhase::Complete; + self.result = Some(result); + } + + pub fn status(&self) -> RemoteRestoreStatus { + let mut status = RemoteRestoreStatus::default(); + status.phase = self.phase; + + match self.phase { + RemoteRestoreTaskPhase::Init => {} + RemoteRestoreTaskPhase::Download => { + let progress = self.download_progress.as_ref().unwrap().get_progress(); + status.download_progress = Some(progress); + } + RemoteRestoreTaskPhase::Unpack => { + let progress = self.unpack_progress.as_ref().unwrap().get_progress(); + status.unpack_progress = Some(progress); + } + RemoteRestoreTaskPhase::Restore => { + let (id, manager) = self.restore_status.as_ref().unwrap(); + let restore_status = manager.get_task_status(id).unwrap(); + status.restore_status = Some(restore_status); + } + RemoteRestoreTaskPhase::Complete => { + status.result = self.result.clone(); + } + } + + status + } +} + +#[derive(Clone)] +pub struct RemoteRestoreStatusManager(Arc>); + +impl RemoteRestoreStatusManager { + pub fn new() -> Self { + Self(Arc::new(Mutex::new(RemoteRestoreStatusManagerInner::new()))) + } + + pub fn begin_download(&self, progress: ArchiveProgressHolder) { + self.0.lock().unwrap().begin_download(progress) + } + + pub fn begin_unpack(&self, progress: ArchiveProgressHolder) { + self.0.lock().unwrap().begin_unpack(progress) + } + + pub fn begin_restore(&self, task_id: &str, restore_manager: RestoreManagerRef) { + self.0 + .lock() + .unwrap() + .begin_restore(task_id, restore_manager) + } + + pub fn complete(&self, result: BuckyResult<()>) { + self.0.lock().unwrap().complete(result) + } + + pub fn status(&self) -> RemoteRestoreStatus { + self.0.lock().unwrap().status() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn display() { + let e = BuckyError::new(BuckyErrorCode::NotFound, "not found error"); + + let mut progress = ArchiveProgress::new(); + progress.current = Some(FileProgress { + file: "object.0.data".to_owned(), + total: 100000, + completed: 100, + result: None, + }); + + let status = RemoteRestoreStatus { + phase: RemoteRestoreTaskPhase::Download, + // result: Some(Ok(())), + result: Some(Err(e)), + + download_progress: Some(progress), + unpack_progress: Some(ArchiveProgress::new()), + restore_status: Some(RestoreStatus::default()), + }; + + let s = serde_json::to_string_pretty(&status).unwrap(); + println!("{}", s); + } +} diff --git a/src/component/cyfs-backup/src/remote_restore/task.rs b/src/component/cyfs-backup/src/remote_restore/task.rs new file mode 100644 index 000000000..606748d31 --- /dev/null +++ b/src/component/cyfs-backup/src/remote_restore/task.rs @@ -0,0 +1,263 @@ +use super::def::*; +use super::status::*; +use crate::archive_download::TaskAbortHandler; +use crate::backup::RestoreManager; +use crate::{archive_download::*, remote_restore::status::RemoteRestoreTaskPhase}; +use cyfs_backup_lib::*; +use cyfs_base::*; + +use futures::future::{AbortHandle, Abortable}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +#[derive(Clone)] +pub struct RemoteRestoreTask { + id: String, + + archive_dir: PathBuf, + archive_file: PathBuf, + + status: RemoteRestoreStatusManager, + + // Use for cancel task + task_handle: Arc>>, + abort_handler: TaskAbortHandler, +} + +impl RemoteRestoreTask { + pub fn new(id: impl Into) -> Self { + let id = id.into(); + + let tmp_dir = cyfs_util::get_temp_path().join("restore"); + let archive_dir = tmp_dir.join(&id); + if archive_dir.is_dir() { + warn!( + "local archive dir exists already! {}", + archive_dir.display() + ); + } + + let archive_file = archive_dir.join("archive"); + + Self { + id: id.into(), + + archive_dir, + archive_file, + + status: RemoteRestoreStatusManager::new(), + + task_handle: Arc::new(Mutex::new(None)), + abort_handler: TaskAbortHandler::new(), + } + } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn status(&self) -> RemoteRestoreStatus { + self.status.status() + } + + pub async fn abort(&self) { + warn!("will cancel restore task: {}", self.id); + + self.abort_handler.abort(); + + if let Some(task_handle) = self.task_handle.lock().unwrap().take() { + task_handle.abort(); + } + + // Wait for task compelte(for some blocking task) + async_std::task::sleep(std::time::Duration::from_secs(5)).await; + + warn!( + "cancel restore task complete, now will clean temp data! {}", + self.id + ); + + self.clean_temp_data().await; + } + + pub async fn run(&self, params: RemoteRestoreParams) -> BuckyResult<()> { + assert_eq!(self.status.status().phase, RemoteRestoreTaskPhase::Init); + + let (abort_handle, abort_registration) = AbortHandle::new_pair(); + + let task = self.clone(); + let fut = Abortable::new( + async move { + let ret = task.run_inner(params).await; + task.status.complete(ret.clone()); + + // FIXME should we clean the archive file and archive dir? + task.clean_temp_data().await; + + ret + }, + abort_registration, + ); + + { + let mut holder = self.task_handle.lock().unwrap(); + assert!(holder.is_none()); + *holder = Some(abort_handle); + } + + match fut.await { + Ok(ret) => ret, + Err(futures::future::Aborted { .. }) => { + let msg = format!("The restore task has been aborted! {}", self.id); + warn!("{}", msg); + Err(BuckyError::new(BuckyErrorCode::Aborted, msg)) + } + } + } + + pub fn start(&mut self, params: RemoteRestoreParams) -> BuckyResult<()> { + let task = self.clone(); + async_std::task::spawn(async move { + let id = params.id.clone(); + match task.run(params).await { + Ok(()) => { + info!("run remote restore task complete! task={}", id); + } + Err(e) => { + error!("run remote restore task failed! task={}, {}", id, e); + } + } + }); + + Ok(()) + } + + async fn run_inner(&self, params: RemoteRestoreParams) -> BuckyResult<()> { + let remote_archive = RemoteArchiveInfo::parse(¶ms.remote_archive).map_err(|e| { + let msg = format!( + "invalid remote archive url format: {}, {}", + params.remote_archive, e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::InvalidFormat, msg) + })?; + + if !self.archive_dir.is_dir() { + if let Err(e) = async_std::fs::create_dir_all(&self.archive_dir).await { + let msg = format!( + "create local archive dir failed! {}, {}", + self.archive_dir.display(), + e + ); + error!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::IoError, msg)); + } + } + + match remote_archive { + RemoteArchiveInfo::ZipFile(file_url) => { + let url = file_url.parse_url()?; + + // First download archive to local file + let progress = ArchiveProgressHolder::new(); + self.status.begin_download(progress.clone()); + + info!( + "will download archive to local file: {} -> {}", + url, + self.archive_file.display() + ); + + let downloader = ArchiveFileDownloader::new(url, self.archive_file.clone()); + downloader.download(&progress).await?; + + // Then unpack the archive to target dir + let progress = ArchiveProgressHolder::new(); + self.status.begin_unpack(progress.clone()); + + info!( + "will extract archive to local dir: {} -> {}", + self.archive_file.display(), + self.archive_dir.display() + ); + + let unzip = ArchiveUnzip::new(self.archive_file.clone(), self.archive_dir.clone()); + let ret = unzip.unzip(&progress, &self.abort_handler).await; + + ret?; + } + RemoteArchiveInfo::Folder(folder_url) => { + let progress = ArchiveProgressHolder::new(); + self.status.begin_download(progress.clone()); + + info!( + "will download archive to local dir: {} -> {}", + folder_url.base_url, + self.archive_dir.display() + ); + + let downloader = ArchiveFolderDownloader::new(folder_url, self.archive_dir.clone()); + downloader.download(&progress).await?; + } + } + + // Create restore task + let restore_manager = Arc::new(RestoreManager::new()); + + let cyfs_root = params.cyfs_root.unwrap_or( + cyfs_util::get_cyfs_root_path_ref() + .as_os_str() + .to_string_lossy() + .to_string(), + ); + let isolate = params.isolate.unwrap_or("".to_owned()); + + let restore_params = UniRestoreParams { + id: params.id, + cyfs_root, + isolate, + archive: self.archive_dir.clone(), + password: params.password, + }; + + self.status + .begin_restore(&restore_params.id, restore_manager.clone()); + + restore_manager.run_uni_restore(restore_params).await?; + + Ok(()) + } + + async fn clean_temp_data(&self) { + // Remove the local file after we unpack + if self.archive_file.is_file() { + if let Err(e) = async_std::fs::remove_file(&self.archive_file).await { + error!( + "remove temp archive file failed! {}, {}", + self.archive_file.display(), + e + ); + } else { + info!( + "remove temp archive file success! {}", + self.archive_file.display() + ); + } + } + + if self.archive_dir.is_dir() { + if let Err(e) = async_std::fs::remove_dir_all(&self.archive_dir).await { + error!( + "remove temp archive dir failed! {}, {}", + self.archive_dir.display(), + e + ); + } else { + info!( + "remove temp archive dir success! {}", + self.archive_dir.display() + ); + } + } + } +} diff --git a/src/component/cyfs-backup/src/remote_restore/test.rs b/src/component/cyfs-backup/src/remote_restore/test.rs new file mode 100644 index 000000000..899156323 --- /dev/null +++ b/src/component/cyfs-backup/src/remote_restore/test.rs @@ -0,0 +1,101 @@ +use cyfs_base::BuckyErrorCode; + +use super::*; + +use std::sync::Arc; + +async fn download_folder() { + let manager = Arc::new(RemoteRestoreManager::new()); + + const ID: &str = "remote_store1"; + let params = RemoteRestoreParams { + id: ID.to_owned(), + + cyfs_root: Some("C://cyfs/tmp/remote_store".to_owned()), + isolate: None, + password: None, + + remote_archive: "http://127.0.0.1:8887/test65/${filename}?token=123456".to_owned(), + }; + + let manager1 = manager.clone(); + async_std::task::spawn(async move { + loop { + async_std::task::sleep(std::time::Duration::from_secs(2)).await; + + let ret = manager1.get_task_status(ID); + match ret { + Ok(status) => { + println!("status: {}", serde_json::to_string(&status).unwrap()); + } + Err(e) => { + if e.code() == BuckyErrorCode::NotFound { + warn!("restore task not found! {}", ID); + break; + } else { + unreachable!(); + } + } + } + } + }); + + // test cancel + let manager1 = manager.clone(); + async_std::task::spawn(async move { + loop { + async_std::task::sleep(std::time::Duration::from_secs(20)).await; + + manager1.abort_task(ID).await.unwrap(); + } + }); + + match manager.run_remote_restore(params).await { + Ok(()) => { + info!("run restore task complete!"); + } + Err(e) => { + match e.code() { + BuckyErrorCode::Aborted => { + warn!("restore task aborted!"); + } + _ => { + unreachable!(); + } + } + } + } +} + +async fn download_file() { + let manager = Arc::new(RemoteRestoreManager::new()); + + const ID: &str = "remote_store1"; + let params = RemoteRestoreParams { + id: ID.to_owned(), + + cyfs_root: Some("C://cyfs/tmp/remote_store".to_owned()), + isolate: None, + password: None, + + remote_archive: "http://127.0.0.1:8887/test65.zip?token=123456".to_owned(), + }; + + let manager1 = manager.clone(); + async_std::task::spawn(async move { + loop { + async_std::task::sleep(std::time::Duration::from_secs(2)).await; + + let status = manager1.get_task_status(ID).unwrap(); + println!("status: {}", serde_json::to_string(&status).unwrap()); + } + }); + + manager.run_remote_restore(params).await.unwrap(); +} + +#[test] +fn test() { + cyfs_base::init_simple_log("test-remote-restore", None); + async_std::task::block_on(download_folder()); +} diff --git a/src/component/cyfs-backup/src/state_backup/helper.rs b/src/component/cyfs-backup/src/state_backup/helper.rs index bc184d2ad..16be7c0e1 100644 --- a/src/component/cyfs-backup/src/state_backup/helper.rs +++ b/src/component/cyfs-backup/src/state_backup/helper.rs @@ -104,7 +104,7 @@ impl ObjectTraverserHandler for ObjectTraverserHelper { return ObjectTraverseFilterResult::Keep(None); } - match self.dec_meta.as_ref().unwrap().query_path_config(path) { + match self.dec_meta.as_ref().unwrap().query_path_config(path).await { Some(meta) => match meta.storage_state { None | Some(GlobalStatePathStorageState::Concrete) => { ObjectTraverseFilterResult::Keep(meta.depth.map(|v| v as u32)) @@ -135,7 +135,7 @@ impl ObjectTraverserHandler for ObjectTraverserHelper { None => object as &dyn ObjectSelectorDataProvider, }; - match self.dec_meta.as_ref().unwrap().query_object_meta(provider) { + match self.dec_meta.as_ref().unwrap().query_object_meta(provider).await { Some(meta) => ObjectTraverseFilterResult::Keep(meta.depth.map(|v| v as u32)), None => ObjectTraverseFilterResult::Keep(None), } diff --git a/src/component/cyfs-base/src/base/access_string.rs b/src/component/cyfs-base/src/base/access_string.rs index 45f32f1da..323f97f6f 100644 --- a/src/component/cyfs-base/src/base/access_string.rs +++ b/src/component/cyfs-base/src/base/access_string.rs @@ -116,9 +116,9 @@ impl TryFrom for AccessPermissions { } } -impl TryFrom<&str> for AccessPermissions { - type Error = BuckyError; - fn try_from(value: &str) -> BuckyResult { +impl FromStr for AccessPermissions { + type Err = BuckyError; + fn from_str(value: &str) -> BuckyResult { match value { "---" => Ok(AccessPermissions::None), "--x" => Ok(AccessPermissions::CallOnly), @@ -157,7 +157,7 @@ impl<'de> Visitor<'de> for AccessPermissionsVisitor { } fn visit_str(self, v: &str) -> Result where E: Error { - AccessPermissions::try_from(v).map_err(Error::custom) + AccessPermissions::from_str(v).map_err(Error::custom) } } @@ -384,7 +384,7 @@ impl FromStr for AccessString { fn from_str(value: &str) -> BuckyResult { let mut access = AccessString::new(0); for (mut chunk, group) in value.chars().filter(|c|c != &'_' && c != &' ').chunks(3).into_iter().zip(ACCESS_GROUP_LIST) { - access.set_group_permissions(*group, AccessPermissions::try_from(chunk.join("").as_str())?); + access.set_group_permissions(*group, AccessPermissions::from_str(chunk.join("").as_str())?); } Ok(access) @@ -435,7 +435,7 @@ impl<'de> Visitor<'de> for AccessStringVisitor { let mut ret = AccessString::default(); while let Some(value) = seq.next_element::()? { ret.set_group_permissions(AccessGroup::try_from(value.group.as_str()).map_err(Error::custom)?, - AccessPermissions::try_from(value.access.as_str()).map_err(Error::custom)?); + AccessPermissions::from_str(value.access.as_str()).map_err(Error::custom)?); } Ok(ret) diff --git a/src/component/cyfs-base/src/objects/object_map/path.rs b/src/component/cyfs-base/src/objects/object_map/path.rs index e7c12562f..bf70eff38 100644 --- a/src/component/cyfs-base/src/objects/object_map/path.rs +++ b/src/component/cyfs-base/src/objects/object_map/path.rs @@ -80,6 +80,7 @@ impl ObjectMapPath { /* /a/b/ -> /a/b / -> / + /a/b?arg=xxx -> /a/b */ fn fix_path(path: &str) -> BuckyResult<&str> { let path = path.trim(); @@ -87,7 +88,13 @@ impl ObjectMapPath { return Ok(path); } - // 末尾的/需要去除 + // Remove the query params + let path = match path.rsplit_once('?') { + Some((path, _)) => path, + None => path, + }; + + // The / at the end needs to be removed let path_ret = path.trim_end_matches("/"); if !path_ret.starts_with("/") { let msg = format!("invalid objectmap path format! path={}", path); @@ -103,7 +110,7 @@ impl ObjectMapPath { let mut current = self.get_root().await?; let path = Self::fix_path(path)?; - // 判断是不是root + // Check if is root path if path == "/" { return Ok(Some(current)); } diff --git a/src/component/cyfs-base/src/objects/object_map/path_env.rs b/src/component/cyfs-base/src/objects/object_map/path_env.rs index d21826cad..8fd79977d 100644 --- a/src/component/cyfs-base/src/objects/object_map/path_env.rs +++ b/src/component/cyfs-base/src/objects/object_map/path_env.rs @@ -444,7 +444,7 @@ impl ObjectMapPathOpEnv { this.path().commit_op_list().await?; } else { // env操作期间，root没发生改变，那么不再重新执行op_list - info!("will clear op list because root not changed: {}", root); + info!("will clear op list because root not changed during the operations: {}", root); this.path().clear_op_list(); } diff --git a/src/component/cyfs-base/src/objects/object_map/single_env.rs b/src/component/cyfs-base/src/objects/object_map/single_env.rs index d78a2bad5..33eb1ab06 100644 --- a/src/component/cyfs-base/src/objects/object_map/single_env.rs +++ b/src/component/cyfs-base/src/objects/object_map/single_env.rs @@ -134,7 +134,7 @@ impl ObjectMapSingleOpEnv { "load single_op_env with inner_path but not found! root={}, inner_path={}", obj_map_id, inner_path, ); - error!("{}", msg); + warn!("{}", msg); return Err(BuckyError::new(BuckyErrorCode::NotFound, msg)); } @@ -171,7 +171,7 @@ impl ObjectMapSingleOpEnv { "load single_op_env by path but not found! root={}, path={}, key={}", root, path, key ); - error!("{}", msg); + warn!("{}", msg); return Err(BuckyError::new(BuckyErrorCode::NotFound, msg)); } diff --git a/src/component/cyfs-bdt/src/debug/command.rs b/src/component/cyfs-bdt/src/debug/command.rs index 8efecd612..76b2ebb3a 100644 --- a/src/component/cyfs-bdt/src/debug/command.rs +++ b/src/component/cyfs-bdt/src/debug/command.rs @@ -8,20 +8,9 @@ use cyfs_base::*; use crate::stack::Stack; pub fn debug_command_line() -> clap::App<'static, 'static> { - App::new("bdt-debug") - .about("bdt stack debug") - .arg(Arg::with_name("host") - .short("h") - .long("host") - .value_name("host") - .help("connect remote host") - .default_value("127.0.0.1")) - .arg(Arg::with_name("port") - .short("p") - .long("port") - .value_name("port") - .help("local server port") - .default_value("12345")) + App::new("bdt-debug").about("bdt stack debug") + .arg(Arg::with_name("host").short("h").long("host").value_name("host").help("connect remote host").default_value("127.0.0.1")) + .arg(Arg::with_name("port").short("p").long("port").value_name("port").help("local server port").default_value("12345")) .subcommand(SubCommand::with_name("test")) .subcommand(SubCommand::with_name("ping") .arg(Arg::with_name("remote").required(true)) diff --git a/src/component/cyfs-bdt/src/debug/mod.rs b/src/component/cyfs-bdt/src/debug/mod.rs index deced2c4b..b8cd12449 100644 --- a/src/component/cyfs-bdt/src/debug/mod.rs +++ b/src/component/cyfs-bdt/src/debug/mod.rs @@ -4,4 +4,4 @@ mod ping; pub use command::{debug_command_line, DebugCommand}; pub use stub::{DebugStub, Config}; -pub use ping::PingStub; +pub use ping::{PingStub, Pinger}; diff --git a/src/component/cyfs-bdt/src/debug/ping.rs b/src/component/cyfs-bdt/src/debug/ping.rs index 9bdf2b352..9337de779 100644 --- a/src/component/cyfs-bdt/src/debug/ping.rs +++ b/src/component/cyfs-bdt/src/debug/ping.rs @@ -17,6 +17,7 @@ use async_std::{ future, }; use std::time::Duration; +use std::io::ErrorKind; struct PingStubImpl { stack: WeakStack, @@ -94,12 +95,18 @@ impl Pinger { let ts = cyfs_base::bucky_time_now(); options.sequence = Some(TempSeq::from(ts as u32)); - if let Err(err) = self.0.datagram_tunnel.send_to( + if let Err(err) = self.0.datagram_tunnel.send_to( buf, &mut options, &remote.desc().device_id(), datagram::ReservedVPort::Debug.into()) { - return Err(BuckyError::new(BuckyErrorCode::CodeError, format!("ping remote={:?} send err={:?}", remote, err))) + match err.kind() { + ErrorKind::NotConnected => { + }, + _ => { + return Err(BuckyError::new(BuckyErrorCode::CodeError, format!("ping remote={:?} send err={:?}", remote, err))); + } + } } match future::timeout(timeout, self.0.datagram_tunnel.recv_v()).await { diff --git a/src/component/cyfs-bdt/src/debug/stub.rs b/src/component/cyfs-bdt/src/debug/stub.rs index 5d57a8efa..e283664c7 100644 --- a/src/component/cyfs-bdt/src/debug/stub.rs +++ b/src/component/cyfs-bdt/src/debug/stub.rs @@ -2,7 +2,7 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr, Shutdown}, path::Path, str::FromStr, - time::{Duration, Instant} + time::{Duration, Instant}, }; use async_std::{ sync::Arc, diff --git a/src/component/cyfs-lib/src/acl/access.rs b/src/component/cyfs-lib/src/acl/access.rs deleted file mode 100644 index 0563d74f4..000000000 --- a/src/component/cyfs-lib/src/acl/access.rs +++ /dev/null @@ -1,49 +0,0 @@ -use cyfs_base::*; - -use std::str::FromStr; - -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum AclAccess { - Accept = 0, - Reject = 1, - Drop = 2, - Pass = 3, -} - -impl AclAccess { - pub fn as_str(&self) -> &str { - match *self { - Self::Accept => "accept", - Self::Reject => "reject", - Self::Drop => "drop", - Self::Pass => "pass", - } - } -} - -impl ToString for AclAccess { - fn to_string(&self) -> String { - self.as_str().to_owned() - } -} - -impl FromStr for AclAccess { - type Err = BuckyError; - - fn from_str(s: &str) -> Result { - let ret = match s { - "accept" => Self::Accept, - "reject" => Self::Reject, - "drop" => Self::Drop, - "pass" => Self::Pass, - - _ => { - let msg = format!("unknown acl access: {}", s); - error!("{}", msg); - return Err(BuckyError::new(BuckyErrorCode::InvalidFormat, msg)); - } - }; - - Ok(ret) - } -} diff --git a/src/component/cyfs-lib/src/acl/action.rs b/src/component/cyfs-lib/src/acl/action.rs index 97d52f79c..d3f30db99 100644 --- a/src/component/cyfs-lib/src/acl/action.rs +++ b/src/component/cyfs-lib/src/acl/action.rs @@ -2,50 +2,6 @@ use cyfs_base::*; use std::str::FromStr; -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum AclDirection { - Any = 0, - - In = 1, - Out = 2, -} - -impl AclDirection { - pub fn as_str(&self) -> &str { - match *self { - Self::Any => "*", - Self::In => "in", - Self::Out => "out", - } - } -} - -impl FromStr for AclDirection { - type Err = BuckyError; - - fn from_str(s: &str) -> Result { - let ret = match s { - "*" => Self::Any, - "in" => Self::In, - "out" => Self::Out, - - _ => { - let msg = format!("unknown acl direction: {}", s); - error!("{}", msg); - return Err(BuckyError::new(BuckyErrorCode::InvalidFormat, msg)); - } - }; - - Ok(ret) - } -} - -impl ToString for AclDirection { - fn to_string(&self) -> String { - self.as_str().to_owned() - } -} - #[derive(Debug, Clone, Eq, PartialEq)] pub enum AclOperationCategory { Both, @@ -54,264 +10,39 @@ pub enum AclOperationCategory { } #[derive(Debug, Clone, Eq, PartialEq)] -pub enum AclOperation { - Any = 0, - - GetObject = 1, - PutObject = 2, - PostObject = 3, - SelectObject = 4, - DeleteObject = 5, - - SignObject = 6, - VerifyObject = 7, - - PutData = 8, - GetData = 9, - DeleteData = 10, - QueryFile = 11, - - // root-state - ReadRootState = 15, - WriteRootState = 16, - - // non/ndn的一些通用操作 - Get = 20, - Put = 21, - Delete = 22, - - Read = 23, - Write = 24, - - // sign+verify - Crypto = 25, +pub enum AclAction { + Accept = 0, + Reject = 1, } -impl AclOperation { - pub fn category(&self) -> AclOperationCategory { - match *self { - Self::Any => AclOperationCategory::Both, - - Self::GetObject - | Self::SelectObject - | Self::VerifyObject - | Self::GetData - | Self::ReadRootState - | Self::Get - | Self::Read - | Self::QueryFile => AclOperationCategory::Read, - - Self::PutObject - | Self::PostObject - | Self::DeleteObject - | Self::SignObject - | Self::PutData - | Self::DeleteData - | Self::WriteRootState - | Self::Put - | Self::Delete - | Self::Write - | Self::Crypto => AclOperationCategory::Write, - } - } - +impl AclAction { pub fn as_str(&self) -> &str { match *self { - Self::Any => "*", - Self::GetObject => "get-object", - Self::PutObject => "put-object", - Self::PostObject => "post-object", - Self::SelectObject => "select-object", - Self::DeleteObject => "delete-object", - - Self::SignObject => "sign-object", - Self::VerifyObject => "verify-object", - - Self::PutData => "put-data", - Self::GetData => "get-data", - Self::DeleteData => "delete-data", - Self::QueryFile => "query-file", - - Self::WriteRootState => "write-root-state", - Self::ReadRootState => "read-root-state", - - Self::Get => "get", - Self::Put => "put", - Self::Delete => "delete", - - Self::Read => "read", - Self::Write => "write", - - Self::Crypto => "crypto", - } - } - - pub fn is_get(&self) -> bool { - match *self { - Self::GetObject | Self::GetData | Self::Get | Self::Any => true, - _ => false, - } - } - - pub fn is_put(&self) -> bool { - match *self { - Self::PutObject | Self::PutData | Self::Put | Self::Any => true, - _ => false, + Self::Accept => "accept", + Self::Reject => "reject", } } - - pub fn is_delete(&self) -> bool { - match *self { - Self::DeleteObject | Self::DeleteData | Self::Delete | Self::Any => true, - _ => false, - } - } - - pub fn is_crypto(&self) -> bool { - match *self { - Self::SignObject | Self::VerifyObject => true, - _ => false, - } - } - - pub fn is_read(&self) -> bool { - match self.category() { - AclOperationCategory::Read | AclOperationCategory::Both => true, - _ => false, - } - } - - pub fn is_write(&self) -> bool { - match self.category() { - AclOperationCategory::Write | AclOperationCategory::Both => true, - _ => false, - } - } -} - -impl FromStr for AclOperation { - type Err = BuckyError; - - fn from_str(s: &str) -> Result { - let ret = match s { - "*" => Self::Any, - "get-object" => Self::GetObject, - "put-object" => Self::PutObject, - "post-object" => Self::PostObject, - "select-object" => Self::SelectObject, - "delete-object" => Self::DeleteObject, - - "sign-object" => Self::SignObject, - "verify-object" => Self::VerifyObject, - "crypto" => Self::Crypto, - - "put-data" => Self::PutData, - "get-data" => Self::GetData, - "delete-data" => Self::DeleteData, - "query-file" => Self::QueryFile, - - "read-root-state" => Self::ReadRootState, - "write-root-state" => Self::WriteRootState, - - "put" => Self::Put, - "get" => Self::Get, - "delete" => Self::Delete, - - "read" => Self::Read, - "write" => Self::Write, - - _ => { - let msg = format!("unknown acl operation: {}", s); - error!("{}", msg); - return Err(BuckyError::new(BuckyErrorCode::InvalidFormat, msg)); - } - }; - - Ok(ret) - } } -impl ToString for AclOperation { +impl ToString for AclAction { fn to_string(&self) -> String { self.as_str().to_owned() } } -#[derive(Debug, Clone)] -pub struct AclAction { - pub direction: AclDirection, - pub operation: AclOperation, -} - -impl Default for AclAction { - fn default() -> Self { - Self { - direction: AclDirection::Any, - operation: AclOperation::Any, - } - } -} - -impl AclAction { - pub fn new(direction: AclDirection, operation: AclOperation) -> Self { - Self { - direction, - operation, - } - } +impl FromStr for AclAction { + type Err = BuckyError; - pub fn is_match(&self, req: &Self) -> bool { - assert_ne!(req.direction, AclDirection::Any); - assert_ne!(req.operation, AclOperation::Any); + fn from_str(s: &str) -> Result { + let ret = match s { + "accept" => Self::Accept, + "reject" => Self::Reject, - match &self.direction { - AclDirection::Any => {} _ => { - if self.direction != req.direction { - return false; - } - } - } - - if self.operation == req.operation { - return true; - } - - // 一些通用操作判断 - match &self.operation { - AclOperation::Any => true, - AclOperation::Get => req.operation.is_get(), - AclOperation::Put => req.operation.is_put(), - AclOperation::Delete => req.operation.is_delete(), - AclOperation::Read => req.operation.is_read(), - AclOperation::Write => req.operation.is_write(), - AclOperation::Crypto => req.operation.is_crypto(), - _ => false, - } - } - - pub fn parse(s: &str) -> BuckyResult { - let ret = if s == "*" { - Self { - direction: AclDirection::Any, - operation: AclOperation::Any, - } - } else { - // 必须由至少两段组成 *-operation/direction-*/direction-operation - let parts: Vec<&str> = s.split('-').collect(); - if parts.len() < 2 { - let msg = format!("invalid action format: {}", s); + let msg = format!("unknown acl action: {}", s); error!("{}", msg); return Err(BuckyError::new(BuckyErrorCode::InvalidFormat, msg)); } - - let direction = AclDirection::from_str(parts[0])?; - let operation = AclOperation::from_str(&parts[1..].join("-"))?; - - Self { - direction, - operation, - } }; Ok(ret) diff --git a/src/component/cyfs-lib/src/acl/handler/request.rs b/src/component/cyfs-lib/src/acl/handler/request.rs index 61610f525..cac9bac3c 100644 --- a/src/component/cyfs-lib/src/acl/handler/request.rs +++ b/src/component/cyfs-lib/src/acl/handler/request.rs @@ -1,52 +1,32 @@ -use super::super::{AclAccess, AclAction}; +use super::super::{AclAction}; use crate::base::*; -use crate::ndn::*; -use crate::non::NONSlimObjectInfo; use cyfs_base::*; +#[derive(Debug, Clone)] pub struct AclHandlerRequest { - // 来源协议 - pub protocol: RequestProtocol, - - // 动作 - pub action: AclAction, + // The request's target dec + pub dec_id: ObjectId, - // source/target - pub device_id: DeviceId, + // request source + pub source: RequestSourceInfo, - // 操作对象 - pub object: Option, - pub inner_path: Option, + // full req_path = {req_path}?{query_string} + pub req_path: String, + pub req_query_string: Option, - // 所属dec - pub dec_id: String, - - // 请求的path - pub req_path: Option, - - // 引用对象 - pub referer_object: Option>, + // The required permissions + pub permissions: AccessPermissions, } impl std::fmt::Display for AclHandlerRequest { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "protocol: {:?}", self.protocol)?; - write!(f, ", action: {:?}", self.action)?; - write!(f, ", device: {}", self.device_id)?; - - if let Some(object) = &self.object { - write!(f, ", object: {:?}", object)?; - } - - if let Some(inner_path) = &self.inner_path { - write!(f, ", inner_path: {}", inner_path)?; - } - - write!(f, ", dec: {}", self.dec_id)?; - write!(f, ", req_path: {:?}", self.req_path)?; - if let Some(referer_object) = &self.referer_object { - write!(f, ", referer_object: {:?}", referer_object)?; + write!(f, "dec_id: {}", self.dec_id)?; + write!(f, ", source: {}", self.source)?; + write!(f, ", req_path: {}", self.req_path)?; + if let Some(v) = &self.req_query_string { + write!(f, ", req_query_string: {}", v)?; } + write!(f, ", permissions: {}", self.permissions)?; Ok(()) } @@ -54,11 +34,11 @@ impl std::fmt::Display for AclHandlerRequest { #[derive(Debug, Clone)] pub struct AclHandlerResponse { - pub access: AclAccess, + pub action: AclAction, } impl std::fmt::Display for AclHandlerResponse { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "access: {:?}", self.access) + write!(f, "action: {:?}", self.action) } } diff --git a/src/component/cyfs-lib/src/acl/handler/request_codec.rs b/src/component/cyfs-lib/src/acl/handler/request_codec.rs index 30ff0d37b..5a5ea310d 100644 --- a/src/component/cyfs-lib/src/acl/handler/request_codec.rs +++ b/src/component/cyfs-lib/src/acl/handler/request_codec.rs @@ -1,62 +1,33 @@ -use super::super::AclAction; use super::request::*; use cyfs_base::*; use serde_json::{Map, Value}; -impl JsonCodec for AclAction { - fn encode_json(&self) -> Map { - let mut obj = Map::new(); - JsonCodecHelper::encode_string_field(&mut obj, "direction", &self.direction); - JsonCodecHelper::encode_string_field(&mut obj, "operation", &self.operation); - - obj - } - - fn decode_json(obj: &Map) -> BuckyResult { - Ok(Self { - direction: JsonCodecHelper::decode_string_field(obj, "direction")?, - operation: JsonCodecHelper::decode_string_field(obj, "operation")?, - }) - } -} impl JsonCodec for AclHandlerRequest { fn encode_json(&self) -> Map { let mut obj = Map::new(); - JsonCodecHelper::encode_string_field(&mut obj, "protocol", &self.protocol); - JsonCodecHelper::encode_field(&mut obj, "action", &self.action); + JsonCodecHelper::encode_string_field(&mut obj, "dec_id", &self.dec_id); + JsonCodecHelper::encode_field(&mut obj, "source", &self.source); - JsonCodecHelper::encode_string_field(&mut obj, "device_id", &self.device_id); + JsonCodecHelper::encode_string_field(&mut obj, "req_path", &self.req_path); + JsonCodecHelper::encode_option_string_field(&mut obj, "req_query_string", self.req_query_string.as_ref()); - JsonCodecHelper::encode_option_field(&mut obj, "object", self.object.as_ref()); + JsonCodecHelper::encode_string_field(&mut obj, "permissions", &self.permissions); - JsonCodecHelper::encode_option_string_field( - &mut obj, - "inner_path", - self.inner_path.as_ref(), - ); - - JsonCodecHelper::encode_option_string_field(&mut obj, "req_path", self.req_path.as_ref()); - JsonCodecHelper::encode_string_field(&mut obj, "dec_id", &self.dec_id); - JsonCodecHelper::encode_option_str_array_field(&mut obj, "referer_object", self.referer_object.as_ref()); - obj } fn decode_json(obj: &Map) -> BuckyResult { Ok(Self { - protocol: JsonCodecHelper::decode_string_field(obj, "protocol")?, - action: JsonCodecHelper::decode_field(obj, "action")?, - device_id: JsonCodecHelper::decode_string_field(obj, "device_id")?, + dec_id: JsonCodecHelper::decode_string_field(obj, "dec_id")?, + source: JsonCodecHelper::decode_field(obj, "source")?, - object: JsonCodecHelper::decode_option_field(obj, "object")?, - inner_path: JsonCodecHelper::decode_option_string_field(obj, "inner_path")?, + req_path: JsonCodecHelper::decode_string_field(obj, "req_path")?, + req_query_string: JsonCodecHelper::decode_option_string_field(obj, "req_query_string")?, - req_path: JsonCodecHelper::decode_option_string_field(obj, "req_path")?, - dec_id: JsonCodecHelper::decode_string_field(obj, "dec_id")?, - referer_object: JsonCodecHelper::decode_option_str_array_field(obj, "referer_object")?, + permissions: JsonCodecHelper::decode_string_field(obj, "permissions")?, }) } } @@ -65,14 +36,14 @@ impl JsonCodec for AclHandlerRequest { impl JsonCodec for AclHandlerResponse { fn encode_json(&self) -> Map { let mut obj = Map::new(); - JsonCodecHelper::encode_string_field(&mut obj, "access", &self.access); + JsonCodecHelper::encode_string_field(&mut obj, "action", &self.action); obj } fn decode_json(obj: &Map) -> BuckyResult { Ok(Self { - access: JsonCodecHelper::decode_string_field(obj, "access")?, + action: JsonCodecHelper::decode_string_field(obj, "action")?, }) } } \ No newline at end of file diff --git a/src/component/cyfs-lib/src/acl/mod.rs b/src/component/cyfs-lib/src/acl/mod.rs index 034e74bf6..c09e45281 100644 --- a/src/component/cyfs-lib/src/acl/mod.rs +++ b/src/component/cyfs-lib/src/acl/mod.rs @@ -1,7 +1,5 @@ mod handler; mod action; -mod access; -pub use access::*; pub use handler::*; pub use action::*; \ No newline at end of file diff --git a/src/component/cyfs-lib/src/base/exp_filter.rs b/src/component/cyfs-lib/src/base/exp_filter.rs index 74780cb82..cca32c9b4 100644 --- a/src/component/cyfs-lib/src/base/exp_filter.rs +++ b/src/component/cyfs-lib/src/base/exp_filter.rs @@ -835,6 +835,13 @@ impl ExpTokenEvalValue { Self::Glob(ExpGlobToken::new_string_list(list)) } + pub fn from_glob(v: &T) -> Self + where + T: ToString, + { + Self::Glob(ExpGlobToken::new_string(v.to_string())) + } + pub fn from_opt_glob(v: &Option) -> Self where T: ToString, diff --git a/src/component/cyfs-lib/src/prelude/global_state_common.rs b/src/component/cyfs-lib/src/prelude/global_state_common.rs index 135653ba2..39b59e81a 100644 --- a/src/component/cyfs-lib/src/prelude/global_state_common.rs +++ b/src/component/cyfs-lib/src/prelude/global_state_common.rs @@ -37,6 +37,9 @@ pub struct RequestGlobalStatePath { // inernal path of global-state, without the dec-id segment pub req_path: Option, + + // extra query params in URL query parameters format, after a question mark (?) + pub req_query_string: Option, } impl fmt::Display for RequestGlobalStatePath { @@ -57,6 +60,7 @@ impl RequestGlobalStatePath { global_state_root: None, dec_id, req_path: req_path.map(|v| v.into()), + req_query_string: None, } } @@ -93,6 +97,28 @@ impl RequestGlobalStatePath { } } + pub fn set_req_query_string(&mut self, query: impl Into) { + self.req_query_string = Some(query.into()); + } + + pub fn parse_req_path_with_query_string( + req_path_with_query_string: &str, + ) -> (&str, Option<&str>) { + match req_path_with_query_string.rsplit_once('?') { + Some((req_path, query_string)) => (req_path, Some(query_string)), + None => (req_path_with_query_string, None), + } + } + + pub fn parse_req_path_with_query_string_owned( + req_path_with_query_string: &str, + ) -> (String, Option) { + let (req_path, query_string) = + Self::parse_req_path_with_query_string(req_path_with_query_string); + + (req_path.to_owned(), query_string.map(|v| v.to_owned())) + } + /* The first paragraph is optional root-state/local-cache, default root-state The second paragraph is optional current/root:{root-id}/dec-root:{dec-root-id}, default is current @@ -100,6 +126,8 @@ impl RequestGlobalStatePath { Fourth paragraph optional global-state-inner-path */ pub fn parse(req_path: &str) -> BuckyResult { + let (req_path, query_string) = Self::parse_req_path_with_query_string(req_path); + let segs: Vec<&str> = req_path .trim_start_matches('/') .split('/') @@ -163,7 +191,7 @@ impl RequestGlobalStatePath { }; let dec_id = if index < segs.len() { - // 如果第一段是object_id，那么认为是dec_id + // If first segment is object_id，then treat as dec_id let seg = segs[index]; if OBJECT_ID_BASE58_RANGE.contains(&seg.len()) { let dec_id = ObjectId::from_str(seg).map_err(|e| { @@ -192,6 +220,7 @@ impl RequestGlobalStatePath { global_state_root, dec_id, req_path, + req_query_string: query_string.map(|s| s.to_string()), }) } @@ -214,7 +243,14 @@ impl RequestGlobalStatePath { segs.push(Cow::Borrowed(path.trim_start_matches('/'))); } - format!("/{}", segs.join("/")) + match &self.req_query_string { + Some(v) => { + format!("/{}?{}", segs.join("/"), v) + } + None => { + format!("/{}", segs.join("/")) + } + } } pub fn match_target(&self, target: &Self) -> bool { @@ -253,6 +289,7 @@ mod test { global_state_root: None, dec_id: None, req_path: Some("/a/b/".to_owned()), + req_query_string: Some("token=123456".to_owned()), }; let s = root.format_string(); @@ -260,10 +297,11 @@ mod test { let r = RequestGlobalStatePath::parse(&s).unwrap(); assert_eq!(root, r); - root.dec_id = Some(ObjectId::default()); + let dec_id = ObjectId::from_str("9tGpLNnSzxs7kX2pbe27adjNjGQTgFzMCR9pDQ4rHRpM").unwrap(); + root.dec_id = Some(dec_id); let s = root.format_string(); println!("{}", s); - + root.global_state_category = Some(GlobalStateCategory::RootState); let s = root.format_string(); println!("{}", s); @@ -299,6 +337,7 @@ mod test { global_state_root: None, dec_id: None, req_path: None, + req_query_string: None, }; let s = root.format_string(); diff --git a/src/component/cyfs-lib/src/rmeta/def/access.rs b/src/component/cyfs-lib/src/rmeta/def/access.rs index 88754365f..8a2f75263 100644 --- a/src/component/cyfs-lib/src/rmeta/def/access.rs +++ b/src/component/cyfs-lib/src/rmeta/def/access.rs @@ -92,6 +92,7 @@ impl PartialOrd for GlobalStatePathSpecifiedGroup { pub enum GlobalStatePathGroupAccess { Specified(GlobalStatePathSpecifiedGroup), Default(u32 /*AccessString*/), + Handler, } impl PartialOrd for GlobalStatePathGroupAccess { @@ -99,12 +100,17 @@ impl PartialOrd for GlobalStatePathGroupAccess { match &self { Self::Specified(left) => match other { Self::Specified(right) => left.partial_cmp(&right), - Self::Default(_) => Some(Ordering::Less), + _ => Some(Ordering::Less), }, Self::Default(_left) => match other { Self::Specified(_) => Some(Ordering::Greater), Self::Default(_right) => Some(Ordering::Equal), + Self::Handler => Some(Ordering::Less), }, + Self::Handler => match other { + Self::Handler => Some(Ordering::Equal), + _ => Some(Ordering::Greater), + } } } } @@ -131,6 +137,9 @@ impl std::fmt::Display for GlobalStatePathGroupAccess { AccessPermissions::format_u8(s.access), ) } + Self::Handler => { + write!(f, "[Handler]") + } } } } @@ -144,6 +153,7 @@ impl GlobalStatePathGroupAccess { return false; } } + Self::Handler => {} } true @@ -229,20 +239,38 @@ impl Ord for GlobalStatePathAccessItem { pub struct GlobalStateAccessRequest<'d, 'a, 'b> { pub dec: Cow<'d, ObjectId>, - pub path: Cow<'a, str>, pub source: Cow<'b, RequestSourceInfo>, + + pub path: Cow<'a, str>, + pub query_string: Option>, + pub permissions: AccessPermissions, } impl<'d, 'a, 'b> std::fmt::Display for GlobalStateAccessRequest<'d, 'a, 'b> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "path={}, {}, permissions={}", - self.path, - self.source, - self.permissions.as_str() - ) + match &self.query_string { + Some(query_string) => { + write!( + f, + "path={}, query={}, {}, permissions={}", + self.path, + query_string, + self.source, + self.permissions.as_str() + ) + } + None => { + write!( + f, + "path={}, {}, permissions={}", + self.path, + self.source, + self.permissions.as_str() + ) + } + } + } } diff --git a/src/component/cyfs-lib/src/rmeta/def/handler.rs b/src/component/cyfs-lib/src/rmeta/def/handler.rs new file mode 100644 index 000000000..3d9559de9 --- /dev/null +++ b/src/component/cyfs-lib/src/rmeta/def/handler.rs @@ -0,0 +1,26 @@ +use crate::RequestSourceInfo; +use cyfs_base::*; + +use std::sync::Arc; + +pub struct GlobalStatePathHandlerRequest { + // target_dec_id + pub dec_id: ObjectId, + + // request source + pub source: RequestSourceInfo, + + // full_req_path = {req_path}?{query_string} + pub req_path: String, + pub req_query_string: Option, + + // The required permissions + pub permissions: AccessPermissions, +} + +#[async_trait::async_trait] +pub trait GlobalStatePathHandler: Sync + Send { + async fn on_check(&self, req: GlobalStatePathHandlerRequest) -> BuckyResult; +} + +pub type GlobalStatePathHandlerRef = Arc>; diff --git a/src/component/cyfs-lib/src/rmeta/def/mod.rs b/src/component/cyfs-lib/src/rmeta/def/mod.rs index 135aa83ee..87364ea60 100644 --- a/src/component/cyfs-lib/src/rmeta/def/mod.rs +++ b/src/component/cyfs-lib/src/rmeta/def/mod.rs @@ -4,10 +4,12 @@ mod def; mod object; mod config; mod path; +mod handler; pub use access::*; pub use link::*; pub use def::*; pub use object::*; pub use config::*; -pub use path::*; \ No newline at end of file +pub use path::*; +pub use handler::*; \ No newline at end of file diff --git a/src/component/cyfs-lib/src/rmeta/processor.rs b/src/component/cyfs-lib/src/rmeta/processor.rs index a08c7094d..17b87af12 100644 --- a/src/component/cyfs-lib/src/rmeta/processor.rs +++ b/src/component/cyfs-lib/src/rmeta/processor.rs @@ -84,9 +84,10 @@ pub trait GlobalStateMetaRawProcessor: Send + Sync { async fn clear_access(&self) -> BuckyResult; - fn check_access<'d, 'a, 'b>( + async fn check_access<'d, 'a, 'b>( &self, req: GlobalStateAccessRequest<'d, 'a, 'b>, + handler: &GlobalStatePathHandlerRef, ) -> BuckyResult<()>; // link relate methods @@ -100,7 +101,7 @@ pub trait GlobalStateMetaRawProcessor: Send + Sync { async fn clear_link(&self) -> BuckyResult; - fn resolve_link(&self, source: &str) -> BuckyResult>; + async fn resolve_link(&self, source: &str) -> BuckyResult>; // object meta async fn add_object_meta(&self, item: GlobalStateObjectMetaItem) -> BuckyResult; @@ -112,7 +113,7 @@ pub trait GlobalStateMetaRawProcessor: Send + Sync { async fn clear_object_meta(&self) -> BuckyResult; - fn check_object_access( + async fn check_object_access( &self, target_dec_id: &ObjectId, object_data: &dyn ObjectSelectorDataProvider, @@ -120,7 +121,7 @@ pub trait GlobalStateMetaRawProcessor: Send + Sync { permissions: AccessPermissions, ) -> BuckyResult>; - fn query_object_meta( + async fn query_object_meta( &self, object_data: &dyn ObjectSelectorDataProvider, ) -> Option; @@ -134,7 +135,7 @@ pub trait GlobalStateMetaRawProcessor: Send + Sync { ) -> BuckyResult>; async fn clear_path_config(&self) -> BuckyResult; - fn query_path_config(&self, path: &str) -> Option; + async fn query_path_config(&self, path: &str) -> Option; } pub type GlobalStateMetaRawProcessorRef = Arc>; diff --git a/src/component/cyfs-lib/src/router_handler/filter.rs b/src/component/cyfs-lib/src/router_handler/filter.rs index 8e66f68d9..f8ed54f48 100644 --- a/src/component/cyfs-lib/src/router_handler/filter.rs +++ b/src/component/cyfs-lib/src/router_handler/filter.rs @@ -71,7 +71,6 @@ impl ExpReservedTokenTranslatorHelper { return Some(v); } - let ret = match token { "req_path" => ExpTokenEvalValue::from_opt_glob(&common.req_path), "level" => ExpTokenEvalValue::from_string(&common.level), @@ -87,7 +86,7 @@ impl ExpReservedTokenTranslatorHelper { fn trans_bdt_interest_referer( token: &str, - referer: &BdtDataRefererInfo + referer: &BdtDataRefererInfo, ) -> Option { let ret = match token { "target" => ExpTokenEvalValue::from_opt_string(&referer.target), @@ -101,7 +100,7 @@ impl ExpReservedTokenTranslatorHelper { } else { ExpTokenEvalValue::None } - }, + } "flags" => ExpTokenEvalValue::U32(referer.flags), _ => { return None; @@ -755,9 +754,7 @@ impl ExpReservedTokenTranslator for CryptoDecryptDataInputResponse { fn trans(&self, token: &str) -> ExpTokenEvalValue { match token { "result" => ExpTokenEvalValue::from_string(&self.result), - _ => { - ExpTokenEvalValue::None - } + _ => ExpTokenEvalValue::None, } } } @@ -765,47 +762,19 @@ impl ExpReservedTokenTranslator for CryptoDecryptDataInputResponse { // acl impl ExpReservedTokenTranslator for AclHandlerRequest { fn trans(&self, token: &str) -> ExpTokenEvalValue { - match token { - "protocol" => ExpTokenEvalValue::from_string(&self.protocol), - "direction" => ExpTokenEvalValue::from_string(&self.action.direction), - "operation" => ExpTokenEvalValue::from_string(&self.action.operation), - - "device" | "source" | "target" | "your" => { - ExpTokenEvalValue::from_string(&self.device_id) - } + if let Some(v) = + ExpReservedTokenTranslatorHelper::trans_request_source(token, &self.source) + { + return v; + } - "inner_path" => ExpTokenEvalValue::from_opt_glob(&self.inner_path), + match token { "dec_id" => ExpTokenEvalValue::from_string(&self.dec_id), - "req_path" => ExpTokenEvalValue::from_opt_glob(&self.req_path), - "referer_object" => { - if let Some(referer_object) = &self.referer_object { - if referer_object.len() > 0 { - ExpTokenEvalValue::from_glob_list(referer_object) - } else { - ExpTokenEvalValue::None - } - } else { - ExpTokenEvalValue::None - } - } + "req_path" => ExpTokenEvalValue::from_glob(&self.req_path), + "req_query_string" => ExpTokenEvalValue::from_opt_glob(&self.req_query_string), + "permissions" => ExpTokenEvalValue::from_string(&self.permissions.as_str()), _ => { - if let Some(v) = ExpReservedTokenTranslatorHelper::trans_object_id( - token, - self.object.as_ref().map(|item| &item.object_id), - ) { - return v; - } - - let object = self - .object - .as_ref() - .map(|item| item.object.as_deref()) - .flatten(); - if let Some(v) = ExpReservedTokenTranslatorHelper::trans_object(token, object) { - return v; - } - unreachable!("unknown router acl request reserved token: {}", token); } } @@ -815,24 +784,25 @@ impl ExpReservedTokenTranslator for AclHandlerRequest { impl ExpReservedTokenTranslator for AclHandlerResponse { fn trans(&self, token: &str) -> ExpTokenEvalValue { match token { - "access" => ExpTokenEvalValue::from_string(&self.access), + "action" => ExpTokenEvalValue::from_string(&self.action.as_str()), _ => ExpTokenEvalValue::None, } } } - // interest impl ExpReservedTokenTranslator for InterestHandlerRequest { fn trans(&self, token: &str) -> ExpTokenEvalValue { match token { - "chunk" => ExpTokenEvalValue::from_string(&self.chunk), - "from" => ExpTokenEvalValue::from_opt_string(&self.from), - "from_channel" => ExpTokenEvalValue::from_string(&self.from_channel), + "chunk" => ExpTokenEvalValue::from_string(&self.chunk), + "from" => ExpTokenEvalValue::from_opt_string(&self.from), + "from_channel" => ExpTokenEvalValue::from_string(&self.from_channel), "group_path" => ExpTokenEvalValue::from_opt_glob(&self.group_path), _ => { if let Some(referer) = &self.referer { - if let Some(v) = ExpReservedTokenTranslatorHelper::trans_bdt_interest_referer(token, referer) { + if let Some(v) = + ExpReservedTokenTranslatorHelper::trans_bdt_interest_referer(token, referer) + { return v; } } @@ -845,38 +815,46 @@ impl ExpReservedTokenTranslator for InterestHandlerRequest { impl ExpReservedTokenTranslator for InterestHandlerResponse { fn trans(&self, token: &str) -> ExpTokenEvalValue { match token { - "type" => ExpTokenEvalValue::String(self.type_str().to_owned()), + "type" => ExpTokenEvalValue::String(self.type_str().to_owned()), "upload.groups" => match self { - Self::Upload { groups, .. } => ExpTokenEvalValue::from_glob_list(&groups), - _ => ExpTokenEvalValue::from_glob_list(&Vec::::new()), - }, + Self::Upload { groups, .. } => ExpTokenEvalValue::from_glob_list(&groups), + _ => ExpTokenEvalValue::from_glob_list(&Vec::::new()), + }, "upload.file_path" => match self { Self::Upload { source, .. } => match source { - InterestUploadSource::ChunkStore => ExpTokenEvalValue::None, - InterestUploadSource::File { path, .. } => ExpTokenEvalValue::from_opt_glob(&path.to_str()) - }, - _ => ExpTokenEvalValue::None, - }, + InterestUploadSource::ChunkStore => ExpTokenEvalValue::None, + InterestUploadSource::File { path, .. } => { + ExpTokenEvalValue::from_opt_glob(&path.to_str()) + } + }, + _ => ExpTokenEvalValue::None, + }, "upload.file_offset" => match self { Self::Upload { source, .. } => match source { - InterestUploadSource::ChunkStore => ExpTokenEvalValue::None, - InterestUploadSource::File { offset, .. } => ExpTokenEvalValue::U64(*offset), - }, - _ => ExpTokenEvalValue::None, - }, - "transmit_to" => ExpTokenEvalValue::from_opt_string(&self.transmit_to().clone()), + InterestUploadSource::ChunkStore => ExpTokenEvalValue::None, + InterestUploadSource::File { offset, .. } => ExpTokenEvalValue::U64(*offset), + }, + _ => ExpTokenEvalValue::None, + }, + "transmit_to" => ExpTokenEvalValue::from_opt_string(&self.transmit_to().clone()), "err" => { if let Some(err) = self.resp_interest().map(|r| r.err.as_u16()) { ExpTokenEvalValue::U32(err as u32) } else { ExpTokenEvalValue::None } - }, - "redirect" => ExpTokenEvalValue::from_opt_string(&self.resp_interest().and_then(|r| r.redirect.clone())), - "redirect_referer_target" => ExpTokenEvalValue::from_opt_string(&self.resp_interest().and_then(|r| r.redirect_referer_target.clone())), + } + "redirect" => ExpTokenEvalValue::from_opt_string( + &self.resp_interest().and_then(|r| r.redirect.clone()), + ), + "redirect_referer_target" => ExpTokenEvalValue::from_opt_string( + &self + .resp_interest() + .and_then(|r| r.redirect_referer_target.clone()), + ), _ => { unreachable!("unknown router interest response reserved token: {}", token); - }, + } } } } @@ -928,7 +906,7 @@ pub struct RouterHandlerReservedTokenList { pub decrypt_data: ExpReservedTokenList, pub acl: ExpReservedTokenList, - pub interest: ExpReservedTokenList, + pub interest: ExpReservedTokenList, } impl RouterHandlerReservedTokenList { @@ -950,7 +928,7 @@ impl RouterHandlerReservedTokenList { decrypt_data: Self::gen_decrypt_object(), acl: Self::gen_acl(), - interest: Self::gen_interest(), + interest: Self::gen_interest(), } } @@ -964,14 +942,18 @@ impl RouterHandlerReservedTokenList { token_list.add_u32("flags"); } - fn add_non_input_request_common_tokens(token_list: &mut ExpReservedTokenList) { - token_list.add_glob("req_path"); - + fn add_request_source_tokens(token_list: &mut ExpReservedTokenList) { token_list.add_string("source.dec_id"); token_list.add_string("source.device"); token_list.add_string("source.zone_category"); token_list.add_string("source.zone"); token_list.add_string("source.protocol"); + } + + fn add_non_input_request_common_tokens(token_list: &mut ExpReservedTokenList) { + token_list.add_glob("req_path"); + + Self::add_request_source_tokens(token_list); token_list.add_string("level"); token_list.add_string("target"); @@ -980,12 +962,8 @@ impl RouterHandlerReservedTokenList { fn add_ndn_input_request_common_tokens(token_list: &mut ExpReservedTokenList) { token_list.add_glob("req_path"); - - token_list.add_string("source.dec_id"); - token_list.add_string("source.device"); - token_list.add_string("source.zone_category"); - token_list.add_string("source.zone"); - token_list.add_string("source.protocol"); + + Self::add_request_source_tokens(token_list); token_list.add_string("level"); token_list.add_glob("referer_object"); @@ -995,13 +973,9 @@ impl RouterHandlerReservedTokenList { fn add_crypto_input_request_common_tokens(token_list: &mut ExpReservedTokenList) { token_list.add_glob("req_path"); - - token_list.add_string("source.dec_id"); - token_list.add_string("source.device"); - token_list.add_string("source.zone_category"); - token_list.add_string("source.zone"); - token_list.add_string("source.protocol"); - + + Self::add_request_source_tokens(token_list); + token_list.add_string("target"); token_list.add_u32("flags"); } @@ -1359,7 +1333,7 @@ impl RouterHandlerReservedTokenList { let mut token_list = ExpReservedTokenList::new(); Self::add_crypto_input_request_common_tokens(&mut token_list); - + token_list.add_u32("crypto_flags"); token_list.add_string("encrypt_type"); @@ -1385,7 +1359,7 @@ impl RouterHandlerReservedTokenList { let mut token_list = ExpReservedTokenList::new(); Self::add_crypto_input_request_common_tokens(&mut token_list); - + token_list.add_u32("crypto_flags"); token_list.add_string("decrypt_type"); @@ -1411,23 +1385,13 @@ impl RouterHandlerReservedTokenList { fn gen_acl_request() -> ExpReservedTokenList { let mut token_list = ExpReservedTokenList::new(); - token_list.add_string("protocol"); - token_list.add_string("direction"); - token_list.add_string("operation"); - - // 下面四个都对应device - token_list.add_string("device"); - token_list.add_string("source"); - token_list.add_string("target"); - token_list.add_string("your"); - - Self::add_object_info_tokens(&mut token_list); - token_list.add_glob("inner_path"); + Self::add_request_source_tokens(&mut token_list); token_list.add_string("dec_id"); token_list.add_glob("req_path"); + token_list.add_glob("req_query_string"); - token_list.add_glob("referer_object"); + token_list.add_string("permissions"); token_list } @@ -1435,7 +1399,7 @@ impl RouterHandlerReservedTokenList { fn gen_acl_response() -> ExpReservedTokenList { let mut token_list = ExpReservedTokenList::new(); - token_list.add_bool("access"); + token_list.add_bool("action"); Self::add_error_tokens(&mut token_list); token_list.translate_resp(); @@ -1453,7 +1417,7 @@ impl RouterHandlerReservedTokenList { // interest fn gen_interest_request() -> ExpReservedTokenList { let mut token_list = ExpReservedTokenList::new(); - + token_list.add_string("chunk"); token_list.add_string("from"); Self::add_bdt_interest_referer_tokens(&mut token_list); @@ -1485,7 +1449,6 @@ impl RouterHandlerReservedTokenList { list } - pub fn select(&self) -> &ExpReservedTokenList where REQ: Send + Sync + 'static + JsonCodec + std::fmt::Display, @@ -1509,7 +1472,7 @@ impl RouterHandlerReservedTokenList { RouterHandlerCategory::EncryptData => &self.encrypt_data, RouterHandlerCategory::DecryptData => &self.decrypt_data, - RouterHandlerCategory::Acl => &self.acl, + RouterHandlerCategory::Acl => &self.acl, RouterHandlerCategory::Interest => &self.interest, } } diff --git a/src/component/cyfs-lib/src/storage/collection.rs b/src/component/cyfs-lib/src/storage/collection.rs index 7c0dd24fa..4654cedb4 100644 --- a/src/component/cyfs-lib/src/storage/collection.rs +++ b/src/component/cyfs-lib/src/storage/collection.rs @@ -96,7 +96,12 @@ impl NOCStorageWrapper { ) -> Self { Self { storage: Box::new(NOCGlobalStateStorage::new( - global_state, dec_id, path, target, id, noc, + global_state, + dec_id, + path, + target, + id, + noc, )), } } @@ -352,9 +357,7 @@ where id: &str, noc: NamedObjectCacheRef, ) -> Self { - let storage = NOCGlobalStateStorage::new( - global_state, dec_id, path, target, id, noc, - ); + let storage = NOCGlobalStateStorage::new(global_state, dec_id, path, target, id, noc); Self { coll: Arc::new(Mutex::new(T::default())), @@ -492,11 +495,18 @@ where } } -pub struct NOCCollectionRWSync -where - T: Default + CollectionCodec + Send + Sync + 'static, -{ - coll: Arc>, + +#[async_trait::async_trait] +pub trait NOCCollectionStorageColl: Send + Sync { + async fn encode(&self) -> BuckyResult>; +} + +pub type NOCCollectionStorageCollRef = Arc>; + +#[derive(Clone)] +pub struct NOCCollectionStorage { + coll: NOCCollectionStorageCollRef, + storage: Arc>, dirty: Arc, @@ -504,29 +514,11 @@ where auto_save: Arc, } - -impl Clone for NOCCollectionRWSync -where - T: Default + CollectionCodec + Send + Sync + 'static, -{ - fn clone(&self) -> Self { - Self { - coll: self.coll.clone(), - storage: self.storage.clone(), - dirty: self.dirty.clone(), - auto_save: self.auto_save.clone(), - } - } -} - -impl NOCCollectionRWSync -where - T: Default + CollectionCodec + Send + Sync + 'static, -{ - pub fn new(id: &str, noc: NamedObjectCacheRef) -> Self { +impl NOCCollectionStorage { + pub fn new(id: &str, coll: NOCCollectionStorageCollRef, noc: NamedObjectCacheRef) -> Self { let noc = NOCRawStorage::new(id, noc); Self { - coll: Arc::new(RwLock::new(T::default())), + coll, storage: Arc::new(Box::new(noc)), dirty: Arc::new(AtomicBool::new(false)), auto_save: Arc::new(AtomicBool::new(false)), @@ -539,14 +531,13 @@ where path: String, target: Option, id: &str, + coll: NOCCollectionStorageCollRef, noc: NamedObjectCacheRef, ) -> Self { - let storage = NOCGlobalStateStorage::new( - global_state, dec_id, path, target, id, noc, - ); + let storage = NOCGlobalStateStorage::new(global_state, dec_id, path, target, id, noc); Self { - coll: Arc::new(RwLock::new(T::default())), + coll, storage: Arc::new(Box::new(storage)), dirty: Arc::new(AtomicBool::new(false)), auto_save: Arc::new(AtomicBool::new(false)), @@ -561,26 +552,11 @@ where self.dirty.store(dirty, Ordering::SeqCst); } - pub fn coll(&self) -> &Arc> { - &self.coll - } - pub fn id(&self) -> &str { self.storage.id() } - pub fn swap(&self, mut value: T) -> T { - { - let mut cur = self.coll.write().unwrap(); - std::mem::swap(&mut *cur, &mut value); - } - - self.set_dirty(true); - - value - } - - pub async fn load(&self) -> BuckyResult<()> { + pub async fn load>(&self) -> BuckyResult> { match self.storage.load().await? { Some(buf) => { let coll = T::decode(&buf).map_err(|e| { @@ -592,10 +568,9 @@ where e })?; - *self.coll.write().unwrap() = coll; - Ok(()) + Ok(Some(coll)) } - None => Ok(()), + None => Ok(None), } } @@ -613,34 +588,23 @@ where } pub async fn save_impl(&self) -> BuckyResult<()> { - let buf = { - let coll = self.coll.read().unwrap(); - coll.encode().map_err(|e| { - error!( - "convert collection to buf failed! id={}, {}", - self.storage.id(), - e - ); - e - })? - }; - + let buf = self.coll.encode().await?; self.storage.save(buf).await } - pub async fn delete(&mut self) -> BuckyResult<()> { + pub async fn delete(&self) -> BuckyResult<()> { self.storage.delete().await?; - // 删除后需要停止自动保存 + // After deleting, we should to stop automatic saving self.stop_save(); - // FIXME 删除后是否要置空? + // FIXME Whether to set it empty after deleting? // self.coll = T::default(); Ok(()) } - // 开始定时保存操作 + // Start to save the operation periodically pub fn start_save(&self, dur: std::time::Duration) { use async_std::prelude::*; @@ -675,3 +639,285 @@ where } } } + +// Collection with std::sync::RWLock +struct RWSyncCollHolder { + coll: Arc>, +} + +impl Clone for RWSyncCollHolder { + fn clone(&self) -> Self { + Self { + coll: self.coll.clone(), + } + } +} + +impl RWSyncCollHolder +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + pub fn new() -> Self { + let coll = Arc::new(std::sync::RwLock::new(T::default())); + + Self { coll } + } + + pub fn into_coll_ref(self) -> NOCCollectionStorageCollRef { + Arc::new(Box::new(self)) + } +} + +#[async_trait::async_trait] +impl NOCCollectionStorageColl for RWSyncCollHolder +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + async fn encode(&self) -> BuckyResult> { + let coll = self.coll.read().unwrap(); + coll.encode().map_err(|e| { + error!("convert collection to buf failed! {}", e); + e + }) + } +} + +#[derive(Clone)] +pub struct NOCCollectionRWSync +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + coll: RWSyncCollHolder, + storage: NOCCollectionStorage, +} + +impl NOCCollectionRWSync +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + pub fn new(id: &str, noc: NamedObjectCacheRef) -> Self { + let coll = RWSyncCollHolder::new(); + let c = coll.clone(); + Self { + coll: c, + storage: NOCCollectionStorage::new(id, coll.into_coll_ref(), noc), + } + } + + pub fn new_global_state( + global_state: GlobalStateOutputProcessorRef, + dec_id: Option, + path: String, + target: Option, + id: &str, + noc: NamedObjectCacheRef, + ) -> Self { + let coll = RWSyncCollHolder::new(); + + Self { + coll: coll.clone(), + storage: NOCCollectionStorage::new_global_state( + global_state, + dec_id, + path, + target, + id, + coll.into_coll_ref(), + noc, + ), + } + } + + pub fn is_dirty(&self) -> bool { + self.storage.is_dirty() + } + + pub fn set_dirty(&self, dirty: bool) { + self.storage.set_dirty(dirty) + } + + pub fn coll(&self) -> &Arc> { + &self.coll.coll + } + + pub fn id(&self) -> &str { + self.storage.id() + } + + pub async fn swap(&self, mut value: T) -> T { + { + let mut cur = self.coll.coll.write().unwrap(); + std::mem::swap(&mut *cur, &mut value); + } + + self.set_dirty(true); + + value + } + + pub async fn load(&self) -> BuckyResult<()> { + if let Some(coll) = self.storage.load().await? { + *self.coll.coll.write().unwrap() = coll; + } + + Ok(()) + } + + pub async fn save(&self) -> BuckyResult<()> { + self.storage.save().await + } + + pub async fn delete(&self) -> BuckyResult<()> { + self.storage.delete().await + } + + // Start to save the operation periodically + pub fn start_save(&self, dur: std::time::Duration) { + self.storage.start_save(dur) + } + + pub fn stop_save(&self) { + self.storage.stop_save() + } +} + +// Collection with async_std::sync::RWLock +struct RWAsyncCollHolder { + coll: Arc>, +} + +impl Clone for RWAsyncCollHolder { + fn clone(&self) -> Self { + Self { + coll: self.coll.clone(), + } + } +} + +impl RWAsyncCollHolder +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + pub fn new() -> Self { + let coll = Arc::new(async_std::sync::RwLock::new(T::default())); + + Self { coll } + } + + pub fn into_coll_ref(self) -> NOCCollectionStorageCollRef { + Arc::new(Box::new(self)) + } +} + +#[async_trait::async_trait] +impl NOCCollectionStorageColl for RWAsyncCollHolder +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + async fn encode(&self) -> BuckyResult> { + let coll = self.coll.read().await; + coll.encode().map_err(|e| { + error!("convert collection to buf failed! {}", e); + e + }) + } +} + +#[derive(Clone)] +pub struct NOCCollectionRWAsync +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + coll: RWAsyncCollHolder, + storage: NOCCollectionStorage, +} + +impl NOCCollectionRWAsync +where + T: Default + CollectionCodec + Send + Sync + 'static, +{ + pub fn new(id: &str, noc: NamedObjectCacheRef) -> Self { + let coll = RWAsyncCollHolder::new(); + let c = coll.clone(); + Self { + coll: c, + storage: NOCCollectionStorage::new(id, coll.into_coll_ref(), noc), + } + } + + pub fn new_global_state( + global_state: GlobalStateOutputProcessorRef, + dec_id: Option, + path: String, + target: Option, + id: &str, + noc: NamedObjectCacheRef, + ) -> Self { + let coll = RWAsyncCollHolder::new(); + + Self { + coll: coll.clone(), + storage: NOCCollectionStorage::new_global_state( + global_state, + dec_id, + path, + target, + id, + coll.into_coll_ref(), + noc, + ), + } + } + + pub fn is_dirty(&self) -> bool { + self.storage.is_dirty() + } + + pub fn set_dirty(&self, dirty: bool) { + self.storage.set_dirty(dirty) + } + + pub fn coll(&self) -> &Arc> { + &self.coll.coll + } + + pub fn id(&self) -> &str { + self.storage.id() + } + + pub async fn swap(&self, mut value: T) -> T { + { + let mut cur = self.coll.coll.write().await; + std::mem::swap(&mut *cur, &mut value); + } + + self.set_dirty(true); + + value + } + + pub async fn load(&self) -> BuckyResult<()> { + if let Some(coll) = self.storage.load().await? { + *self.coll.coll.write().await = coll; + } + + Ok(()) + } + + pub async fn save(&self) -> BuckyResult<()> { + self.storage.save().await + } + + pub async fn delete(&self) -> BuckyResult<()> { + self.storage.delete().await + } + + // Start to save the operation periodically + pub fn start_save(&self, dur: std::time::Duration) { + self.storage.start_save(dur) + } + + pub fn stop_save(&self) { + self.storage.stop_save() + } +} diff --git a/src/component/cyfs-lib/src/storage/state_storage.rs b/src/component/cyfs-lib/src/storage/state_storage.rs index 8c51e01e3..8bd330284 100644 --- a/src/component/cyfs-lib/src/storage/state_storage.rs +++ b/src/component/cyfs-lib/src/storage/state_storage.rs @@ -16,6 +16,44 @@ struct StorageOpData { current: Arc>>, } +struct StorageOpDataHolder { + op_data: StorageOpData, + keep_alive: Option>, +} + +impl StorageOpDataHolder { + fn start_keep_alive(&mut self) { + let path_stub = self.op_data.path_stub.clone(); + let single_stub = self.op_data.single_stub.clone(); + let task = async_std::task::spawn(async move { + loop { + async_std::task::sleep(std::time::Duration::from_secs(60 * 15)).await; + + if let Err(e) = path_stub.get_current_root().await { + error!("path-op-env stub keep alive but failed! {}", e); + } + + if let Err(e) = single_stub.get_current_root().await { + error!("single-op-env stub keep alive but failed! {}", e); + } + } + }); + + assert!(self.keep_alive.is_none()); + self.keep_alive = Some(task); + } + + async fn stop_keep_alive(&mut self, path: &str) { + if let Some(task) = self.keep_alive.take() { + info!("will stop state storage's op-env keep alive! path={}", path); + task.cancel().await; + } else { + warn!("stop state storage's op-env keep alive task but not found! path={}", path); + } + } +} + + pub struct StateStorage { path: String, content_type: ObjectMapSimpleContentType, @@ -25,7 +63,9 @@ pub struct StateStorage { dirty: Arc, auto_save: Arc, - op_data: OnceCell, + + + op_data: OnceCell, } impl Drop for StateStorage { @@ -88,7 +128,7 @@ impl StateStorage { } pub fn stub(&self) -> &SingleOpEnvStub { - &self.op_data.get().unwrap().single_stub + &self.op_data.get().unwrap().op_data.single_stub } pub fn is_dirty(&self) -> bool { @@ -124,7 +164,7 @@ impl StateStorage { let auto_save = self.auto_save.clone(); let path = self.path.clone(); let dirty = self.dirty.clone(); - let op_data = self.op_data.get().unwrap().clone(); + let op_data = self.op_data.get().unwrap().op_data.clone(); async_std::task::spawn(async move { let mut interval = async_std::stream::interval(dur); @@ -148,7 +188,7 @@ impl StateStorage { } } - async fn load(&self) -> BuckyResult { + async fn load(&self) -> BuckyResult { let dec_id = match &self.dec_id { Some(dec_id) => Some(dec_id.to_owned()), None => Some(cyfs_core::get_system_dec_app().to_owned()), @@ -180,12 +220,18 @@ impl StateStorage { current: Arc::new(AsyncMutex::new(current)), }; - Ok(op_data) + let mut holder = StorageOpDataHolder { + op_data, + keep_alive: None, + }; + holder.start_keep_alive(); + + Ok(holder) } // reload the target object and ignore all the unsaved changes! pub async fn reload(&self) -> BuckyResult { - let op_data = self.op_data.get().unwrap(); + let op_data = &self.op_data.get().unwrap().op_data; let new = op_data.path_stub.get_by_path(&self.path).await?; @@ -208,8 +254,8 @@ impl StateStorage { } pub async fn save(&self) -> BuckyResult<()> { - if let Some(op_data) = self.op_data.get() { - Self::save_impl(&self.path, &self.dirty, op_data).await + if let Some(holder) = self.op_data.get() { + Self::save_impl(&self.path, &self.dirty, &holder.op_data).await } else { Ok(()) } @@ -237,15 +283,20 @@ impl StateStorage { pub async fn abort(&mut self) { self.stop_save(); - if let Some(op_data) = self.op_data.take() { - self.abort_impl(op_data).await; + if let Some(holder) = self.op_data.take() { + self.abort_impl(holder).await; } } - async fn abort_impl(&self, op_data: StorageOpData) { + async fn abort_impl(&self, mut holder: StorageOpDataHolder) { info!("will abort state storage: path={}", self.path); - // first hold the lock for update + // First should stop keep alive + holder.stop_keep_alive(&self.path).await; + + let op_data = holder.op_data; + + // Before abord we should first hold the lock for update let mut _current = op_data.current.lock().await; if let Err(e) = op_data.single_stub.abort().await { @@ -539,3 +590,34 @@ impl StateStorageSet { Ok(list) } } + + +#[cfg(test)] +mod test { + + async fn test_keep_alive() { + let task = async_std::task::spawn(async move { + let mut index= 0 ; + loop { + async_std::task::sleep(std::time::Duration::from_secs(1)).await; + + println!("keep alive: {}", index); + index += 1; + } + }); + + async_std::task::sleep(std::time::Duration::from_secs(5)).await; + println!("will cancel keep alive!"); + task.cancel().await; + println!("end cancel keep alive!"); + + async_std::task::sleep(std::time::Duration::from_secs(5)).await; + } + + #[test] + fn test() { + async_std::task::block_on(async move { + test_keep_alive().await; + }) + } +} \ No newline at end of file diff --git a/src/component/cyfs-stack/src/crypto_api/handler/helper.rs b/src/component/cyfs-stack/src/crypto_api/handler/helper.rs index 5a9bfbb46..f79c5f4cc 100644 --- a/src/component/cyfs-stack/src/crypto_api/handler/helper.rs +++ b/src/component/cyfs-stack/src/crypto_api/handler/helper.rs @@ -91,8 +91,8 @@ impl RequestHandlerHelper for CryptoSignObjectInpu self.object.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -115,8 +115,8 @@ impl RequestHandlerHelper for CryptoVerifyObject self.object.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -140,8 +140,8 @@ impl RequestHandlerHelper for CryptoEncryptDataInputRequest { self.data_len().to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -165,8 +165,8 @@ impl RequestHandlerHelper for CryptoDecryptDataInputRequest { self.data.len().to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { diff --git a/src/component/cyfs-stack/src/front/protocol.rs b/src/component/cyfs-stack/src/front/protocol.rs index fff691b30..fa9ed7597 100644 --- a/src/component/cyfs-stack/src/front/protocol.rs +++ b/src/component/cyfs-stack/src/front/protocol.rs @@ -820,6 +820,7 @@ impl FrontProtocolHandler { let mut group = None; let pairs = req.request.url().query_pairs(); + let mut user_pairs = vec![]; for (k, v) in pairs { match k.as_ref() { "mode" => { @@ -863,11 +864,24 @@ impl FrontProtocolHandler { group = Some(RequestorHelper::decode_url_param_with_utf8_decoding(k, v)?); } _ => { - warn!("unknown global state access url query: {}={}", k, v); + debug!("user global state access url query: {}={}", k, v); + user_pairs.push(format!("{}={}", k, v)); } } } + let inner_path: Option = if let Some(inner_path) = inner_path { + if user_pairs.is_empty() { + Some(inner_path) + } else { + let user_querys = user_pairs.join("&"); + Some(format!("{}?{}", inner_path, user_querys)) + } + } else { + None + }; + + let r_req = FrontRRequest { source: req.source, @@ -1075,3 +1089,19 @@ impl FrontProtocolHandler { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_o_query_string() { + let url = "http://www.cyfs.com/a/b?req_path=/a/b%3Ftoken=xxx%26id=xxx&dec_id=xxx"; + let url = http_types::Url::parse(url).unwrap(); + let value: String = RequestorHelper::value_from_querys_with_utf8_decoding("dec_id", &url).unwrap().unwrap(); + assert_eq!(value, "xxx"); + + let value: String = RequestorHelper::value_from_querys_with_utf8_decoding("req_path", &url).unwrap().unwrap(); + assert_eq!(value, "/a/b?token=xxx&id=xxx"); + } +} \ No newline at end of file diff --git a/src/component/cyfs-stack/src/ndn_api/handler/helper.rs b/src/component/cyfs-stack/src/ndn_api/handler/helper.rs index ab30998a0..f0daa481d 100644 --- a/src/component/cyfs-stack/src/ndn_api/handler/helper.rs +++ b/src/component/cyfs-stack/src/ndn_api/handler/helper.rs @@ -68,8 +68,8 @@ impl RequestHandlerHelper for NDNPutDataInputRequest { self.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -92,8 +92,8 @@ impl RequestHandlerHelper for NDNGetDataInputRequest { self.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -116,8 +116,8 @@ impl RequestHandlerHelper for NDNDeleteDataInputReque self.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -142,8 +142,8 @@ impl RequestHandlerHelper for InterestHandlerRequest { self.session_id.value().to_string() } - fn req_path(&self) -> &Option { - &None + fn req_path(&self) -> Option<&String> { + None } fn source(&self) -> &RequestSourceInfo { diff --git a/src/component/cyfs-stack/src/non_api/handler/helper.rs b/src/component/cyfs-stack/src/non_api/handler/helper.rs index 5e44e581c..911066231 100644 --- a/src/component/cyfs-stack/src/non_api/handler/helper.rs +++ b/src/component/cyfs-stack/src/non_api/handler/helper.rs @@ -1,6 +1,5 @@ -use cyfs_lib::*; use cyfs_base::*; - +use cyfs_lib::*; struct RequestUtil; @@ -97,7 +96,7 @@ pub(crate) trait RequestHandlerHelper { fn debug_info(&self) -> String { unimplemented!(); } - fn req_path(&self) -> &Option { + fn req_path(&self) -> Option<&String> { unimplemented!(); } fn source(&self) -> &RequestSourceInfo { @@ -115,8 +114,8 @@ impl RequestHandlerHelper for NONPutObjectInputRequest self.object.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -139,8 +138,8 @@ impl RequestHandlerHelper for NONGetObjectInputRequest self.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -163,8 +162,8 @@ impl RequestHandlerHelper for NONPostObjectInputReque self.object.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -188,8 +187,8 @@ impl RequestHandlerHelper for NONSelectObjectInputR format!("{}", self.filter) } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -212,8 +211,8 @@ impl RequestHandlerHelper for NONDeleteObjectInputR self.object_id.to_string() } - fn req_path(&self) -> &Option { - &self.common.req_path + fn req_path(&self) -> Option<&String> { + self.common.req_path.as_ref() } fn source(&self) -> &RequestSourceInfo { @@ -234,12 +233,10 @@ where { fn update(&mut self, handler: Self) { match self { - Ok(v) => { - match handler { - Ok(new) => v.update(new), - Err(e) => *self = Err(e), - } - } + Ok(v) => match handler { + Ok(new) => v.update(new), + Err(e) => *self = Err(e), + }, Err(_) => { *self = handler; } diff --git a/src/component/cyfs-stack/src/non_api/router/router.rs b/src/component/cyfs-stack/src/non_api/router/router.rs index 0930d9894..e2acdd305 100644 --- a/src/component/cyfs-stack/src/non_api/router/router.rs +++ b/src/component/cyfs-stack/src/non_api/router/router.rs @@ -128,7 +128,7 @@ impl NONRouter { async fn resolve_router_info( &self, - op: AclOperation, + op: AclOperationCategory, source: &RequestSourceInfo, target: Option<&ObjectId>, ) -> BuckyResult { @@ -180,7 +180,7 @@ impl NONRouter { // 计算下一跳的device fn next_forward_target( &self, - op: AclOperation, + op: AclOperationCategory, current_info: &Arc, target_zone_info: &TargetZoneInfo, ) -> BuckyResult> { @@ -208,7 +208,7 @@ impl NONRouter { ) } else { // 判断是不是需要绕过当前zone的ood设备，直接发送请求到目标ood - let bypass_current_ood = match op.category() { + let bypass_current_ood = match op { AclOperationCategory::Read => self.acl.config().read_bypass_ood, AclOperationCategory::Write => self.acl.config().write_bypass_ood, AclOperationCategory::Both => unreachable!(), @@ -341,7 +341,7 @@ impl NONRouter { let router_info = self .resolve_router_info( - AclOperation::PutObject, + AclOperationCategory::Write, &req.common.source, req.common.target.as_ref(), ) @@ -623,7 +623,7 @@ impl NONRouter { let router_info = self .resolve_router_info( - AclOperation::GetObject, + AclOperationCategory::Read, &req.common.source, req.common.target.as_ref(), ) @@ -640,7 +640,7 @@ impl NONRouter { let router_info = self .resolve_router_info( - AclOperation::PostObject, + AclOperationCategory::Write, &req.common.source, req.common.target.as_ref(), ) @@ -758,7 +758,7 @@ impl NONRouter { let router_info = self .resolve_router_info( - AclOperation::GetObject, + AclOperationCategory::Write, &req.common.source, req.common.target.as_ref(), ) diff --git a/src/component/cyfs-stack/src/rmeta_api/object/meta.rs b/src/component/cyfs-stack/src/rmeta_api/object/meta.rs index e195bf97a..dea2a2309 100644 --- a/src/component/cyfs-stack/src/rmeta_api/object/meta.rs +++ b/src/component/cyfs-stack/src/rmeta_api/object/meta.rs @@ -18,6 +18,12 @@ pub(crate) struct ObjectMeta { impl ObjectMeta { pub fn new(item: GlobalStateObjectMetaItem) -> BuckyResult { + if item.access == GlobalStatePathGroupAccess::Handler { + let msg = format!("object meta does not support dynamic handler! {}", item); + error!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::NotSupport, msg)); + } + let selector = ObjectSelector::new(item.selector)?; Ok(Self { selector, @@ -80,6 +86,7 @@ impl GlobalStateObjectMetaList { } else { info!("new object meta: {}", item); self.list.push(item); + self.list.sort(); } true @@ -207,6 +214,11 @@ impl GlobalStateObjectMetaList { continue; } } + GlobalStatePathGroupAccess::Handler => { + let msg = format!("object meta does not support dynamic handler!"); + error!("{}", msg); + continue; + } } } diff --git a/src/component/cyfs-stack/src/rmeta_api/path/access.rs b/src/component/cyfs-stack/src/rmeta_api/path/access.rs index 3a6c4dd7d..a867b465a 100644 --- a/src/component/cyfs-stack/src/rmeta_api/path/access.rs +++ b/src/component/cyfs-stack/src/rmeta_api/path/access.rs @@ -70,7 +70,12 @@ impl GlobalStatePathAccessList { self.list.clone() } - pub fn check<'d, 'a, 'b>(&self, req: GlobalStateAccessRequest<'d, 'a, 'b>, current_device_id: &DeviceId) -> BuckyResult<()> { + pub async fn check<'d, 'a, 'b>( + &self, + req: GlobalStateAccessRequest<'d, 'a, 'b>, + current_device_id: &DeviceId, + handler: &GlobalStatePathHandlerRef, + ) -> BuckyResult<()> { let req_path = if req.path.ends_with('/') { req.path.clone() } else { @@ -86,8 +91,10 @@ impl GlobalStatePathAccessList { info!("raccess match item: req={}, access={}", req, item); return Ok(()); } else { - let msg = - format!("raccess reject by item: device={}, req={}, access={}", current_device_id, req, item); + let msg = format!( + "raccess reject by item: device={}, req={}, access={}", + current_device_id, req, item + ); warn!("{}", msg); return Err(BuckyError::new(BuckyErrorCode::PermissionDenied, msg)); } @@ -99,8 +106,10 @@ impl GlobalStatePathAccessList { info!("raccess match item: req={}, access={}", req, item); return Ok(()); } else { - let msg = - format!("raccess reject by item: device={}, req={}, access={}", current_device_id, req, item); + let msg = format!( + "raccess reject by item: device={}, req={}, access={}", + current_device_id, req, item + ); warn!("{}", msg); return Err(BuckyError::new(BuckyErrorCode::PermissionDenied, msg)); } @@ -109,11 +118,37 @@ impl GlobalStatePathAccessList { continue; } } + GlobalStatePathGroupAccess::Handler => { + let handler_req = GlobalStatePathHandlerRequest { + dec_id: req.dec.as_ref().to_owned(), + source: req.source.as_ref().to_owned(), + req_path: req.path.as_ref().to_owned(), + req_query_string: req.query_string.as_ref().map(|v| v.to_string()), + permissions: req.permissions, + }; + + match handler.on_check(handler_req).await? { + true => { + return Ok(()); + } + false => { + let msg = format!( + "raccess reject by handler: device={}, req={}, access={}", + current_device_id, req, item + ); + warn!("{}", msg); + return Err(BuckyError::new(BuckyErrorCode::PermissionDenied, msg)); + } + } + } } } } - let msg = format!("raccess reject by default: device={}, req={}", current_device_id, req); + let msg = format!( + "raccess reject by default: device={}, req={}", + current_device_id, req + ); warn!("{}", msg); Err(BuckyError::new(BuckyErrorCode::PermissionDenied, msg)) } @@ -124,6 +159,8 @@ mod test_path_access { use super::*; use cyfs_core::*; + use std::sync::Arc; + fn new_dec(name: &str) -> ObjectId { let owner_id = PeopleId::default(); let dec_id = DecApp::generate_id(owner_id.into(), name); @@ -133,10 +170,25 @@ mod test_path_access { dec_id } + struct DummyHandler {} + + #[async_trait::async_trait] + impl GlobalStatePathHandler for DummyHandler { + async fn on_check(&self, req: GlobalStatePathHandlerRequest) -> BuckyResult { + unreachable!(); + } + } + #[test] fn test() { + async_std::task::block_on(test_run()); + } + + async fn test_run() { cyfs_base::init_simple_log("test_path_access", None); + let handler = Arc::new(Box::new(DummyHandler {}) as Box); + let current_device_id = DeviceId::default(); let owner_dec = new_dec("owner"); let mut list = GlobalStatePathAccessList::new(); @@ -191,12 +243,13 @@ mod test_path_access { let ret = GlobalStateAccessRequest { path: Cow::Owned("/d/a/c/".to_owned()), + query_string: None, dec: Cow::Owned(owner_dec.clone()), source: Cow::Borrowed(&source), permissions: AccessPermissions::WriteOnly, }; - list.check(ret, ¤t_device_id).unwrap(); + list.check(ret, ¤t_device_id, &handler).await.unwrap(); // same zone, diff dec let source = RequestSourceInfo { @@ -212,12 +265,13 @@ mod test_path_access { let ret = GlobalStateAccessRequest { path: Cow::Owned("/d/a/c/".to_owned()), + query_string: None, dec: Cow::Owned(owner_dec.clone()), source: Cow::Borrowed(&source), permissions: AccessPermissions::ReadOnly, }; - list.check(ret, ¤t_device_id).unwrap(); + list.check(ret, ¤t_device_id, &handler).await.unwrap(); // same zone, diff dec, write let source = RequestSourceInfo { @@ -233,12 +287,13 @@ mod test_path_access { let ret = GlobalStateAccessRequest { path: Cow::Owned("/d/a/c/".to_owned()), + query_string: None, dec: Cow::Owned(owner_dec.clone()), source: Cow::Borrowed(&source), permissions: AccessPermissions::WriteOnly, }; - list.check(ret, ¤t_device_id).unwrap_err(); + list.check(ret, ¤t_device_id, &handler).await.unwrap_err(); // test remove let device = DeviceId::default(); diff --git a/src/component/cyfs-stack/src/rmeta_api/path/dec_manager.rs b/src/component/cyfs-stack/src/rmeta_api/path/dec_manager.rs index 9339fa12e..8a396db52 100644 --- a/src/component/cyfs-stack/src/rmeta_api/path/dec_manager.rs +++ b/src/component/cyfs-stack/src/rmeta_api/path/dec_manager.rs @@ -92,7 +92,7 @@ impl GlobalStateDecPathMetaHolder { GlobalStateCategory::LocalCache => "cyfs-local-cache-path-meta", }; - let data = NOCCollectionRWSync::::new_global_state( + let data = NOCCollectionRWAsync::::new_global_state( root_state, dec_id.clone(), meta_path, @@ -115,7 +115,7 @@ impl GlobalStateDecPathMetaHolder { "load global state meta success! dec={}, category={}, content={}", GlobalStatePathMetaStorage::get_dec_string(&dec_id), category, - serde_json::to_string(&data.coll().read().unwrap() as &GlobalStatePathMeta).unwrap(), + serde_json::to_string(&data.coll().read().await as &GlobalStatePathMeta).unwrap(), ); let ret = GlobalStatePathMetaSyncCollection::new(device_id, storage, data); diff --git a/src/component/cyfs-stack/src/rmeta_api/path/handler.rs b/src/component/cyfs-stack/src/rmeta_api/path/handler.rs new file mode 100644 index 000000000..6179c33ef --- /dev/null +++ b/src/component/cyfs-stack/src/rmeta_api/path/handler.rs @@ -0,0 +1,91 @@ +use crate::non_api::{NONHandlerCaller, RequestHandlerHelper}; +use crate::router_handler::*; +use cyfs_base::*; +use cyfs_lib::*; + +use std::sync::Arc; + +// acl +impl RequestHandlerHelper for AclHandlerRequest { + fn update(&mut self, handler: Self) { + self.req_path = handler.req_path; + self.dec_id = handler.dec_id; + self.permissions = handler.permissions; + } + + fn debug_info(&self) -> String { + self.req_path.clone() + } + + fn req_path(&self) -> Option<&String> { + Some(&self.req_path) + } + + fn source(&self) -> &RequestSourceInfo { + &self.source + } +} +impl RequestHandlerHelper for AclHandlerResponse { + fn update(&mut self, handler: Self) { + self.action = handler.action; + } +} + +pub struct AclHandlerWrapper { + handlers: Arc, +} + +impl AclHandlerWrapper { + pub fn new(router_handlers: &RouterHandlersManager) -> Self { + Self { + handlers: router_handlers.handlers(&RouterHandlerChain::Acl).clone(), + } + } +} + +#[async_trait::async_trait] +impl GlobalStatePathHandler for AclHandlerWrapper { + async fn on_check(&self, req: GlobalStatePathHandlerRequest) -> BuckyResult { + + // If the target dec_id is not match request's source dec_id, we should set target dec_id into req_path to find the correct handler + let req_path = if req.dec_id != req.source.dec { + format!("{}/{}", req.dec_id, req.req_path.trim_start_matches('/')) + } else { + req.req_path + }; + + let request = AclHandlerRequest { + dec_id: req.dec_id, + source: req.source, + req_path, + req_query_string: req.req_query_string, + permissions: req.permissions, + }; + + // info!("will call acl handler: {:?}", request); + + let mut param = RouterHandlerAclRequest { + request, + response: None, + }; + + if let Some(handler) = self.handlers.try_acl() { + if !handler.is_empty() { + let mut handler = NONHandlerCaller::new(handler.emitter()); + if let Some(ret) = handler.call("acl", &mut param).await? { + let ret = match ret { + Ok(resp) => match resp.action { + AclAction::Accept => Ok(true), + AclAction::Reject => Ok(false), + }, + Err(e) => Err(e), + }; + + return ret; + } + } + } + + Ok(false) + } +} diff --git a/src/component/cyfs-stack/src/rmeta_api/path/meta.rs b/src/component/cyfs-stack/src/rmeta_api/path/meta.rs index db84dd827..b276b04b9 100644 --- a/src/component/cyfs-stack/src/rmeta_api/path/meta.rs +++ b/src/component/cyfs-stack/src/rmeta_api/path/meta.rs @@ -37,7 +37,7 @@ pub struct GlobalStatePathMetaSyncCollection { // current device id device_id: DeviceId, - meta: Arc>, + meta: Arc>, // dump to local file for debug and review storage: Arc, @@ -47,7 +47,7 @@ impl GlobalStatePathMetaSyncCollection { pub fn new( device_id: DeviceId, storage: Arc, - meta: NOCCollectionRWSync, + meta: NOCCollectionRWAsync, ) -> Self { Self { device_id, @@ -60,9 +60,9 @@ impl GlobalStatePathMetaSyncCollection { Arc::new(Box::new(self)) } - fn dump(&self) { + async fn dump(&self) { let data = { - let meta = self.meta.coll().read().unwrap(); + let meta = self.meta.coll().read().await; serde_json::to_string(&meta as &GlobalStatePathMeta).unwrap() }; @@ -78,7 +78,7 @@ impl GlobalStatePathMetaSyncCollection { } { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; let ret = meta.access.add(item); if !ret { return Ok(false); @@ -88,7 +88,7 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(true) } @@ -98,7 +98,7 @@ impl GlobalStatePathMetaSyncCollection { item: GlobalStatePathAccessItem, ) -> BuckyResult> { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.access.remove(item) }; @@ -109,14 +109,14 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } pub async fn clear_access(&self) -> BuckyResult { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.access.clear() }; @@ -127,17 +127,18 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } - pub fn check_access<'d, 'a, 'b>( + pub async fn check_access<'d, 'a, 'b>( &self, req: GlobalStateAccessRequest<'d, 'a, 'b>, + handler: &GlobalStatePathHandlerRef, ) -> BuckyResult<()> { - let meta = self.meta.coll().read().unwrap(); - meta.access.check(req, &self.device_id) + let meta = self.meta.coll().read().await; + meta.access.check(req, &self.device_id, handler).await } pub async fn add_link( @@ -146,7 +147,7 @@ impl GlobalStatePathMetaSyncCollection { target: impl Into + AsRef, ) -> BuckyResult { { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; let ret = meta.link.add(source, target)?; if !ret { return Ok(false); @@ -156,14 +157,14 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(true) } pub async fn remove_link(&self, source: &str) -> BuckyResult> { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.link.remove(source)? }; @@ -174,14 +175,14 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } pub async fn clear_link(&self) -> BuckyResult { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.link.clear() }; @@ -192,13 +193,13 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } - pub fn resolve_link(&self, source: &str) -> BuckyResult> { - let meta = self.meta.coll().read().unwrap(); + pub async fn resolve_link(&self, source: &str) -> BuckyResult> { + let meta = self.meta.coll().read().await; meta.link.resolve(source) } @@ -212,7 +213,7 @@ impl GlobalStatePathMetaSyncCollection { } { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; let ret = meta.object.add(item); if !ret { return Ok(false); @@ -222,7 +223,7 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(true) } @@ -234,7 +235,7 @@ impl GlobalStatePathMetaSyncCollection { let item = ObjectMeta::new_uninit(item); let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.object.remove(&item) }; @@ -245,7 +246,7 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; let item = ret.unwrap(); let ret = GlobalStateObjectMetaItem { @@ -259,7 +260,7 @@ impl GlobalStatePathMetaSyncCollection { pub async fn clear_object_meta(&self) -> BuckyResult { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.object.clear() }; @@ -270,19 +271,19 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } - pub fn check_object_access( + pub async fn check_object_access( &self, target_dec_id: &ObjectId, object_data: &dyn ObjectSelectorDataProvider, source: &RequestSourceInfo, permissions: AccessPermissions, ) -> BuckyResult> { - let meta = self.meta.coll().read().unwrap(); + let meta = self.meta.coll().read().await; meta.object.check( target_dec_id, object_data, @@ -292,11 +293,11 @@ impl GlobalStatePathMetaSyncCollection { ) } - pub fn query_object_meta( + pub async fn query_object_meta( &self, object_data: &dyn ObjectSelectorDataProvider, ) -> Option { - let meta = self.meta.coll().read().unwrap(); + let meta = self.meta.coll().read().await; meta.object .query_object_meta(object_data) .map(|ret| GlobalStateObjectMetaConfigItemValue { @@ -308,7 +309,7 @@ impl GlobalStatePathMetaSyncCollection { // path config pub async fn add_path_config(&self, item: GlobalStatePathConfigItem) -> BuckyResult { { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; let ret = meta.config.add(item); if !ret { return Ok(false); @@ -318,7 +319,7 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(true) } @@ -328,7 +329,7 @@ impl GlobalStatePathMetaSyncCollection { item: GlobalStatePathConfigItem, ) -> BuckyResult> { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.config.remove(item) }; @@ -339,14 +340,14 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } pub async fn clear_path_config(&self) -> BuckyResult { let ret = { - let mut meta = self.meta.coll().write().unwrap(); + let mut meta = self.meta.coll().write().await; meta.config.clear() }; @@ -357,14 +358,14 @@ impl GlobalStatePathMetaSyncCollection { self.meta.set_dirty(true); self.meta.save().await?; - self.dump(); + self.dump().await; Ok(ret) } - pub fn query_path_config(&self, path: &str) -> Option { + pub async fn query_path_config(&self, path: &str) -> Option { let ret = { - let meta = self.meta.coll().write().unwrap(); + let meta = self.meta.coll().write().await; match meta.config.query(path) { Some(item) => Some(GlobalStatePathConfigItemValue { storage_state: item.storage_state, @@ -395,11 +396,12 @@ impl GlobalStateMetaRawProcessor for GlobalStatePathMetaSyncCollection { Self::clear_access(self).await } - fn check_access<'d, 'a, 'b>( + async fn check_access<'d, 'a, 'b>( &self, req: GlobalStateAccessRequest<'d, 'a, 'b>, + handler: &GlobalStatePathHandlerRef, ) -> BuckyResult<()> { - Self::check_access(self, req) + Self::check_access(self, req, handler).await } // link relate methods @@ -415,8 +417,8 @@ impl GlobalStateMetaRawProcessor for GlobalStatePathMetaSyncCollection { Self::clear_link(self).await } - fn resolve_link(&self, source: &str) -> BuckyResult> { - Self::resolve_link(self, source) + async fn resolve_link(&self, source: &str) -> BuckyResult> { + Self::resolve_link(self, source).await } // object meta @@ -435,21 +437,21 @@ impl GlobalStateMetaRawProcessor for GlobalStatePathMetaSyncCollection { Self::clear_object_meta(self).await } - fn query_object_meta( + async fn query_object_meta( &self, object_data: &dyn ObjectSelectorDataProvider, ) -> Option { - Self::query_object_meta(self, object_data) + Self::query_object_meta(self, object_data).await } - fn check_object_access( + async fn check_object_access( &self, target_dec_id: &ObjectId, object_data: &dyn ObjectSelectorDataProvider, source: &RequestSourceInfo, permissions: AccessPermissions, ) -> BuckyResult> { - Self::check_object_access(self, target_dec_id, object_data, source, permissions) + Self::check_object_access(self, target_dec_id, object_data, source, permissions).await } // path config @@ -467,7 +469,7 @@ impl GlobalStateMetaRawProcessor for GlobalStatePathMetaSyncCollection { Self::clear_path_config(self).await } - fn query_path_config(&self, path: &str) -> Option { - Self::query_path_config(self, path) + async fn query_path_config(&self, path: &str) -> Option { + Self::query_path_config(self, path).await } } diff --git a/src/component/cyfs-stack/src/rmeta_api/path/mod.rs b/src/component/cyfs-stack/src/rmeta_api/path/mod.rs index 0010d13fa..82d455002 100644 --- a/src/component/cyfs-stack/src/rmeta_api/path/mod.rs +++ b/src/component/cyfs-stack/src/rmeta_api/path/mod.rs @@ -4,7 +4,9 @@ mod dec_manager; mod link; mod meta; mod storage; +mod handler; pub use meta::*; pub use dec_manager::*; -pub use access::*; \ No newline at end of file +pub use access::*; +pub use handler::*; \ No newline at end of file diff --git a/src/component/cyfs-stack/src/rmeta_api/service/service.rs b/src/component/cyfs-stack/src/rmeta_api/service/service.rs index d663f9bda..e89050e9a 100644 --- a/src/component/cyfs-stack/src/rmeta_api/service/service.rs +++ b/src/component/cyfs-stack/src/rmeta_api/service/service.rs @@ -4,12 +4,14 @@ use super::default::GlobalStateDefaultMetas; use crate::forward::ForwardProcessorManager; use crate::meta::ObjectFailHandler; use crate::rmeta::*; -use crate::rmeta_api::GlobalStatePathMetaSyncCollection; +use crate::rmeta_api::{AclHandlerWrapper, GlobalStatePathMetaSyncCollection}; use crate::root_state_api::GlobalStateLocalService; +use crate::router_handler::RouterHandlersManager; use crate::zone::ZoneManagerRef; use cyfs_base::*; use cyfs_lib::*; +use once_cell::sync::OnceCell; use std::borrow::Cow; use std::sync::Arc; @@ -19,6 +21,8 @@ pub struct GlobalStateMetaLocalService { root_state_meta: GlobalStatePathMetaManagerRef, local_cache_meta: GlobalStatePathMetaManagerRef, + + acl_handler: Arc>, } impl GlobalStateMetaLocalService { @@ -57,6 +61,17 @@ impl GlobalStateMetaLocalService { device_id, root_state_meta, local_cache_meta, + acl_handler: Arc::new(OnceCell::new()), + } + } + + pub(crate) fn init_acl_handler(&self, router_handlers: &RouterHandlersManager) { + // acl handler + let acl_handler = AclHandlerWrapper::new(&router_handlers); + let acl_handler = Arc::new(Box::new(acl_handler) as Box); + + if let Err(_) = self.acl_handler.set(acl_handler) { + unreachable!(); } } @@ -74,9 +89,7 @@ impl GlobalStateMetaLocalService { } } - pub(crate) fn clone_raw_processor( - &self, - ) -> GlobalStateMetaManagerRawProcessorRef { + pub(crate) fn clone_raw_processor(&self) -> GlobalStateMetaManagerRawProcessorRef { Arc::new(Box::new(self.clone())) } @@ -127,11 +140,15 @@ impl GlobalStateMetaLocalService { let check_req = GlobalStateAccessRequest { dec: Cow::Borrowed(target_dec_id), path: req_path.req_path(), + query_string: req_path.req_query_string.as_ref().map(|v| Cow::Borrowed(v.as_str())), source: Cow::Borrowed(source), permissions, }; - if let Err(e) = dec_rmeta.check_access(check_req) { + if let Err(e) = dec_rmeta + .check_access(check_req, self.acl_handler.get().unwrap()) + .await + { error!( "global check access but been rejected! source={}, req_path={}, permissons={}", source, @@ -163,7 +180,10 @@ impl GlobalStateMetaLocalService { let dec_rmeta = ret.unwrap(); let permissions = permissions.into(); - match dec_rmeta.check_object_access(&target_dec_id, object_data, source, permissions) { + match dec_rmeta + .check_object_access(&target_dec_id, object_data, source, permissions) + .await + { Ok(ret) => Ok(ret), Err(e) => { error!( @@ -204,7 +224,8 @@ impl GlobalStateMetaManagerRawProcessor for GlobalStateMetaLocalService { rmeta .get_option_global_state_meta(dec_id, auto_create) - .await.map(|ret| ret.map(|ret| ret.into_processor())) + .await + .map(|ret| ret.map(|ret| ret.into_processor())) } } @@ -262,9 +283,7 @@ impl GlobalStateMetaService { self.local_service.get_meta_manager(category) } - pub(crate) fn clone_manager_raw_processor( - &self, - ) -> GlobalStateMetaManagerRawProcessorRef { + pub(crate) fn clone_manager_raw_processor(&self) -> GlobalStateMetaManagerRawProcessorRef { self.local_service.clone_raw_processor() } diff --git a/src/component/cyfs-stack/src/root_state_api/acl/acl.rs b/src/component/cyfs-stack/src/root_state_api/acl/acl.rs index 927fa57fc..c5a919b59 100644 --- a/src/component/cyfs-stack/src/root_state_api/acl/acl.rs +++ b/src/component/cyfs-stack/src/root_state_api/acl/acl.rs @@ -39,11 +39,14 @@ impl GlobalStateAccessorAclInputProcessor { } } + let (req_path, req_query_string) = RequestGlobalStatePath::parse_req_path_with_query_string_owned(path); + let global_state = RequestGlobalStatePath { global_state_category: None, global_state_root: None, dec_id: Some(dec_id.to_owned()), - req_path: Some(path.to_owned()), + req_path: Some(req_path), + req_query_string, }; self.acl diff --git a/src/component/cyfs-stack/src/root_state_api/acl/zone.rs b/src/component/cyfs-stack/src/root_state_api/acl/zone.rs index 0e1cec700..528544680 100644 --- a/src/component/cyfs-stack/src/root_state_api/acl/zone.rs +++ b/src/component/cyfs-stack/src/root_state_api/acl/zone.rs @@ -53,6 +53,7 @@ impl GlobalStateInputProcessor for GlobalStateAclZoneInputProcessor { global_state_root: None, dec_id: Some(cyfs_core::get_system_dec_app().to_owned()), req_path: Some(CYFS_GLOBAL_STATE_ROOT_VIRTUAL_PATH.to_owned()), + req_query_string: None, } } RootStateRootType::Dec => { @@ -61,6 +62,7 @@ impl GlobalStateInputProcessor for GlobalStateAclZoneInputProcessor { global_state_root: None, dec_id: req.common.target_dec_id.clone(), req_path: None, // None will treat as / + req_query_string: None, } } }; @@ -99,11 +101,13 @@ impl GlobalStateInputProcessor for GlobalStateAclZoneInputProcessor { let access = req.access.as_ref().unwrap(); + let (req_path, req_query_string) = RequestGlobalStatePath::parse_req_path_with_query_string_owned(&access.path); let global_state = RequestGlobalStatePath { global_state_category: Some(self.get_category()), global_state_root: None, dec_id: req.common.target_dec_id.clone(), - req_path: Some(access.path.clone()), + req_path: Some(req_path), + req_query_string, }; self.acl @@ -157,11 +161,13 @@ impl OpEnvInputProcessor for OpEnvAclInnerInputProcessor { .source .check_target_dec_permission(&req.common.target_dec_id) { + let (req_path, req_query_string) = RequestGlobalStatePath::parse_req_path_with_query_string_owned(&req.path); let global_state = RequestGlobalStatePath { global_state_category: Some(self.next.get_category()), global_state_root: None, dec_id: req.common.target_dec_id.clone(), - req_path: Some(req.path.clone()), + req_path: Some(req_path), + req_query_string, }; self.acl diff --git a/src/component/cyfs-stack/src/root_state_api/core/root_index.rs b/src/component/cyfs-stack/src/root_state_api/core/root_index.rs index 208f3b3f1..f33c71940 100644 --- a/src/component/cyfs-stack/src/root_state_api/core/root_index.rs +++ b/src/component/cyfs-stack/src/root_state_api/core/root_index.rs @@ -167,11 +167,6 @@ impl GlobalRootIndex { new_root_id: ObjectId, prev_root_id: Option, ) -> BuckyResult<()> { - info!( - "will update global root: category={}, {:?} -> {}", - self.category, prev_root_id, new_root_id - ); - let _update_lock = self.update_lock.lock().await; let current_root_info; { @@ -187,6 +182,12 @@ impl GlobalRootIndex { root_info.root_state = Some(new_root_id); root_info.revision += 1; + + info!( + "update global root: category={}, {:?} -> {}, revision={}", + self.category, prev_root_id, root_info.root_state.as_ref().unwrap(), root_info.revision, + ); + current_root_info = root_info.clone(); } diff --git a/src/component/cyfs-stack/src/root_state_api/core/test.rs b/src/component/cyfs-stack/src/root_state_api/core/test.rs index de5ab9031..ba5372f17 100644 --- a/src/component/cyfs-stack/src/root_state_api/core/test.rs +++ b/src/component/cyfs-stack/src/root_state_api/core/test.rs @@ -1,8 +1,8 @@ use super::state_manager::*; use crate::config::StackGlobalConfig; -use crate::stack::{CyfsStackParams}; -use cyfs_bdt_ext::*; +use crate::stack::CyfsStackParams; use cyfs_base::*; +use cyfs_bdt_ext::*; use cyfs_core::*; use cyfs_lib::*; @@ -156,7 +156,7 @@ impl NamedObjectCache for MemoryNOC { &self, req: &NamedObjectCacheSelectObjectRequest, ) -> BuckyResult { - unreachable!(); + unreachable!(); } fn bind_object_meta_access_provider( @@ -197,29 +197,33 @@ async fn create_global_state_manager() -> GlobalStateManager { known_device: vec![], known_passive_pn: vec![], udp_sn_only: None, - sn_mode: SNMode::Default, + sn_mode: SNMode::Normal, ping_interval: None, }; let config = StackGlobalConfig::new(params, bdt_params); - let state_manager = GlobalStateManager::load( - GlobalStateCategory::RootState, - &device_id, - Some(owner), - noc.clone(), - config.clone(), - ) - .await - .unwrap(); + let state_manager = GlobalStateManager::new(noc.clone(), config.clone()); + state_manager.load().await.unwrap(); config.change_access_mode(GlobalStateCategory::RootState, GlobalStateAccessMode::Write); state_manager } -async fn test1(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { - let root = global_state_manager +async fn get_root_manager( + global_state_manager: &GlobalStateManager, + dec_id: &ObjectId, +) -> ObjectMapRootManagerRef { + let device_id = DeviceId::from_str("5aSixgPXvhR4puWzFCHqvUXrjFWjxbq4y3thJVgZg6ty").unwrap(); + global_state_manager + .get_root_state(device_id.object_id()) + .await + .unwrap() .get_dec_root_manager(&dec_id, true) .await - .unwrap(); + .unwrap() +} + +async fn test1(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { + let root = get_root_manager(global_state_manager, dec_id).await; // 这里使用非托管模式env let op_env = root.create_op_env(None).unwrap(); @@ -247,24 +251,21 @@ async fn test1(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { // 提交 let dec_root = op_env.commit().await.unwrap(); info!("dec root changed to {}", dec_root); - info!( - "global root changed to {}", - global_state_manager.get_current_root().0 - ); + info!("global root changed to {}", root.get_current_root()); // single op env, test load_with_inner_path let op_env = root.create_single_op_env(None).unwrap(); - op_env.load_with_inner_path(&dec_root, Some("/a/b".to_owned())).await.unwrap(); + op_env + .load_with_inner_path(&dec_root, Some("/a/b".to_owned())) + .await + .unwrap(); let b_value = op_env.get_by_key("test1").await.unwrap().unwrap(); assert_eq!(b_value, x1_value2); } async fn test2(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; // 一组测试数据 let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); @@ -314,10 +315,7 @@ async fn test2(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { } async fn test_update(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { - let root_manager = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root_manager = get_root_manager(global_state_manager, dec_id).await; // 这里使用非托管模式env let op_env = root_manager.create_op_env(None).unwrap(); @@ -382,17 +380,11 @@ async fn test_update(global_state_manager: &GlobalStateManager, dec_id: &ObjectI let current_value = op_env3.get_by_key(path, "test1").await.unwrap(); assert_eq!(current_value, Some(x2_value)); - info!( - "global root changed to {}", - global_state_manager.get_current_root().0 - ); + info!("global root changed to {}", root_manager.get_current_root()); } async fn test_single_env(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); let x1_value2 = ObjectId::from_str("95RvaS5aZKKM8ghTYmsTyhSEWD4pAmALoUSJx1yNxSx5").unwrap(); @@ -440,10 +432,7 @@ async fn test_single_env(global_state_manager: &GlobalStateManager, dec_id: &Obj .unwrap(); let new_root = op_env.commit().await.unwrap(); info!("dec root changed to {}", new_root); - info!( - "global root changed to {}", - global_state_manager.get_current_root().0 - ); + info!("global root changed to {}", root.get_current_root()); // 使用一个新的path_op_env， 校验/a/b/test1的值 let op_env = root.create_op_env(None).unwrap(); @@ -454,11 +443,8 @@ async fn test_single_env(global_state_manager: &GlobalStateManager, dec_id: &Obj } async fn test_isolate_path_env(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { - let root_manager = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); - + let root_manager = get_root_manager(global_state_manager, dec_id).await; + let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); let x1_value2 = ObjectId::from_str("95RvaS5aZKKM8ghTYmsTyhSEWD4pAmALoUSJx1yNxSx5").unwrap(); @@ -466,12 +452,21 @@ async fn test_isolate_path_env(global_state_manager: &GlobalStateManager, dec_id let path_env = root_manager.create_isolate_path_op_env(None).unwrap(); path_env.get_by_path("/a/b").await.unwrap_err(); - path_env.create_new(ObjectMapSimpleContentType::Map).await.unwrap(); + path_env + .create_new(ObjectMapSimpleContentType::Map, None, None) + .await + .unwrap(); path_env.insert_with_path("/a/b", &x1_value).await.unwrap(); - let ret = path_env.set_with_path("/a/b", &x1_value, &None, false).await.unwrap(); + let ret = path_env + .set_with_path("/a/b", &x1_value, &None, false) + .await + .unwrap(); assert_eq!(ret, Some(x1_value)); - let ret = path_env.set_with_path("/a/b", &x1_value2, &Some(x1_value), false).await.unwrap(); + let ret = path_env + .set_with_path("/a/b", &x1_value2, &Some(x1_value), false) + .await + .unwrap(); assert_eq!(ret, Some(x1_value)); let ret = path_env.get_by_path("/a/b").await.unwrap(); assert_eq!(ret, Some(x1_value2)); @@ -480,7 +475,10 @@ async fn test_isolate_path_env(global_state_manager: &GlobalStateManager, dec_id path_env.insert_with_path("/a/c", &x1_value).await.unwrap(); - path_env.create_new_with_path("/s", ObjectMapSimpleContentType::Set).await.unwrap(); + path_env + .create_new_with_path("/s", ObjectMapSimpleContentType::Set) + .await + .unwrap(); path_env.insert("/s", &x1_value).await.unwrap(); path_env.insert("/s", &x1_value2).await.unwrap(); @@ -521,10 +519,7 @@ async fn test_isolate_path_env(global_state_manager: &GlobalStateManager, dec_id async fn test_managed(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { info!("will test managed op_env for dec={}", dec_id); - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; // 一组测试数据 let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); @@ -582,10 +577,7 @@ async fn test_managed(global_state_manager: &GlobalStateManager, dec_id: &Object async fn test_conflict(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { info!("will test conflict op_env for dec={}", dec_id); - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; // 一组测试数据 let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); @@ -642,10 +634,7 @@ async fn test_conflict(global_state_manager: &GlobalStateManager, dec_id: &Objec async fn test_merge(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { info!("will test merge op_env for dec={}", dec_id); - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; // 一组测试数据 let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); @@ -700,10 +689,7 @@ async fn test_merge(global_state_manager: &GlobalStateManager, dec_id: &ObjectId async fn test_path_lock(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { info!("will test path lock op_env for dec={}", dec_id); - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; // 一组测试数据 let x1_value = ObjectId::from_str("95RvaS5anntyAoRUBi48vQoivWzX95M8xm4rkB93DdSt").unwrap(); @@ -769,10 +755,7 @@ async fn test_path_lock(global_state_manager: &GlobalStateManager, dec_id: &Obje } async fn test_remove_panic(global_state_manager: &GlobalStateManager, dec_id: &ObjectId) { - let root = global_state_manager - .get_dec_root_manager(&dec_id, true) - .await - .unwrap(); + let root = get_root_manager(global_state_manager, dec_id).await; let env = root.create_op_env(None).unwrap(); diff --git a/src/component/cyfs-stack/src/root_state_api/local/accessor_service.rs b/src/component/cyfs-stack/src/root_state_api/local/accessor_service.rs index 7bc9edfe6..6e9bdec65 100644 --- a/src/component/cyfs-stack/src/root_state_api/local/accessor_service.rs +++ b/src/component/cyfs-stack/src/root_state_api/local/accessor_service.rs @@ -12,14 +12,8 @@ pub struct GlobalStateAccessorService { } impl GlobalStateAccessorService { - pub fn new( - root_state: GlobalStateRef, - noc: NamedObjectCacheRef, - ) -> Self { - Self { - root_state, - noc, - } + pub fn new(root_state: GlobalStateRef, noc: NamedObjectCacheRef) -> Self { + Self { root_state, noc } } pub fn clone_processor(&self) -> GlobalStateAccessorInputProcessorRef { @@ -90,9 +84,7 @@ impl GlobalStateAccessorService { let object_resp = match object_id.obj_type_code() { ObjectTypeCode::Chunk => NONGetObjectInputResponse::new(object_id, vec![], None), - _ if object_id.is_data() => { - NONGetObjectInputResponse::new(object_id, vec![], None) - } + _ if object_id.is_data() => NONGetObjectInputResponse::new(object_id, vec![], None), _ => { let ret = if object_id.obj_type_code() == ObjectTypeCode::ObjectMap { let ret = root_cache.get_object_map(&object_id).await?; diff --git a/src/component/cyfs-stack/src/root_state_api/service/handler.rs b/src/component/cyfs-stack/src/root_state_api/service/handler.rs index f67c50b65..19009b0d4 100644 --- a/src/component/cyfs-stack/src/root_state_api/service/handler.rs +++ b/src/component/cyfs-stack/src/root_state_api/service/handler.rs @@ -5,6 +5,7 @@ use cyfs_base::*; use cyfs_lib::*; use http_types::StatusCode; +use std::borrow::Cow; use std::str::FromStr; use tide::Response; @@ -995,6 +996,7 @@ impl GlobalStateAccessorRequestHandler { let mut action = GlobalStateAccessorAction::GetObjectByPath; let pairs = req.request.url().query_pairs(); + let mut user_pairs = vec![]; for (k, v) in pairs { match k.as_ref() { "action" => { @@ -1017,13 +1019,20 @@ impl GlobalStateAccessorRequestHandler { page_size = Some(v); } _ => { - warn!("unknown global state accessor url query: {}={}", k, v); + user_pairs.push(format!("{}={}", k, v)); } } } let inner_path = req.request.param("inner_path").unwrap_or("/"); - let inner_path = RequestorHelper::decode_utf8("inner_path", inner_path)?; + let inner_path = if user_pairs.is_empty() { + Cow::Borrowed(inner_path) + } else { + let user_querys = user_pairs.join("&"); + Cow::Owned(format!("{}?{}", inner_path, user_querys)) + }; + + let inner_path = RequestorHelper::decode_utf8("inner_path", &inner_path)?; let inner_path: String = if inner_path.starts_with("/") { inner_path.to_string() diff --git a/src/component/cyfs-stack/src/router_handler/handler_manager.rs b/src/component/cyfs-stack/src/router_handler/handler_manager.rs index a4de09ad9..a68fec33f 100644 --- a/src/component/cyfs-stack/src/router_handler/handler_manager.rs +++ b/src/component/cyfs-stack/src/router_handler/handler_manager.rs @@ -694,7 +694,7 @@ impl RouterHandlersManager { req_path: &Option, filter: &Option, ) -> BuckyResult<()> { - let req_path = if chain == RouterHandlerChain::Handler { + let req_path = if chain == RouterHandlerChain::Handler || chain == RouterHandlerChain::Acl { // Handler must specified valid req_path if req_path.is_none() { let msg = format!( diff --git a/src/component/cyfs-stack/src/stack/cyfs_stack.rs b/src/component/cyfs-stack/src/stack/cyfs_stack.rs index 0d6317e93..bfd914ae7 100644 --- a/src/component/cyfs-stack/src/stack/cyfs_stack.rs +++ b/src/component/cyfs-stack/src/stack/cyfs_stack.rs @@ -261,6 +261,8 @@ impl CyfsStackImpl { error!("load router handlers error! {}", e); } + local_global_state_meta.init_acl_handler(&router_handlers); + // events let router_events = RouterEventsManager::new(); diff --git a/src/component/cyfs-util/src/process/daemon.rs b/src/component/cyfs-util/src/process/daemon.rs index f4b1c9911..7fa44fa2e 100644 --- a/src/component/cyfs-util/src/process/daemon.rs +++ b/src/component/cyfs-util/src/process/daemon.rs @@ -103,8 +103,8 @@ pub mod daemon { .stderr(Stdio::null()); match cmd.spawn() { - Ok(_) => { - info!("spawn as daemon success: {}", cmd_line); + Ok(child) => { + info!("spawn as daemon success: {}, pid={}", cmd_line, child.id()); Ok(()) } diff --git a/src/service/app-manager/Cargo.toml b/src/service/app-manager/Cargo.toml index 9bfecf328..fd18fe2c0 100644 --- a/src/service/app-manager/Cargo.toml +++ b/src/service/app-manager/Cargo.toml @@ -44,4 +44,5 @@ clap = '2.34.0' wait-timeout = '0.2.0' once_cell = "1.17.0" surf = { version = '2.3.2', default-features = false, features = ['h1-client-rustls'] } -itertools = "0.10" \ No newline at end of file +itertools = "0.10" +sysinfo = "0.28" \ No newline at end of file diff --git a/src/service/app-manager/src/app_controller.rs b/src/service/app-manager/src/app_controller.rs index 260017320..91a50c70e 100644 --- a/src/service/app-manager/src/app_controller.rs +++ b/src/service/app-manager/src/app_controller.rs @@ -17,6 +17,7 @@ use std::time::Duration; use async_std::prelude::StreamExt; use once_cell::sync::OnceCell; use app_manager_lib::AppManagerConfig; +use crate::process_util::{get_install_pid_file_path, try_stop_process_by_pid}; pub type AppActionResult = Result; @@ -144,6 +145,17 @@ impl AppController { error!("get app {} owner id failed", &app_id); SubErrorCode::LoadFailed })?; + // stop prev install pid + let use_docker = self.config.app_use_docker(app_id); + info!("app {} use docker install: {}", app_id, use_docker); + if use_docker { + let container_name = format!("decapp-{}-install", app_id.to_string().to_lowercase()); + let _ = stop_docker(&container_name); + } else { + let install_pid_path = get_install_pid_file_path(app_id); + let work_dir = get_app_dir(&app_id.to_string()); + let _ = try_stop_process_by_pid(&install_pid_path, Some(&work_dir)); + } let web_dir_id = AppPackage::install(&app_id, version, &source_id, &owner_id, self.named_cache_client.get().unwrap(), @@ -170,8 +182,7 @@ impl AppController { })?; //run docker install -> build image - let use_docker = self.config.app_use_docker(app_id); - info!("app {} use docker install: {}", app_id, use_docker); + if use_docker { info!("run docker install!"); let id = app_id.to_string(); @@ -184,7 +195,8 @@ impl AppController { SubErrorCode::DockerFailed })?; } else { - let ret = dapp.install(); + let install_pid_path = get_install_pid_file_path(app_id); + let ret = dapp.install(Some(&install_pid_path)); if ret.is_err() || !ret.unwrap() { warn!("exec install command failed. app:{}", app_id); return Err(SubErrorCode::CommondFailed); diff --git a/src/service/app-manager/src/app_manager_ex.rs b/src/service/app-manager/src/app_manager_ex.rs index 27957ce89..fd9b9abc9 100644 --- a/src/service/app-manager/src/app_manager_ex.rs +++ b/src/service/app-manager/src/app_manager_ex.rs @@ -338,65 +338,28 @@ impl AppManager { let fix_status = match status_code { AppLocalStatusCode::Stopping => { info!("find app {} status {} on startup, try stop again", app_id, status_code); - if let Err(e) = self.cmd_executor.as_ref().unwrap().execute_stop( - status_a.clone(), - &AppCmd::stop(self.owner.clone(), app_id.clone()), - 0).await { - warn!("stop app {} on startup failed, err {}", &app_id, e); - Some(AppLocalStatusCode::StopFailed) - } else { - None - } + self.push_front_cmd(&vec![(AppCmd::stop(self.owner.clone(), app_id.clone()), 0)]).await; + None }, AppLocalStatusCode::Starting => { info!("find app {} status {} on startup, try start again", app_id, status_code); - match self.cmd_executor.as_ref().unwrap().execute_start( - status_a.clone(), - &AppCmd::start(self.owner.clone(), app_id.clone()), - 0).await { - Ok(_) => { - None - } - Err(e) => { - error!("start app {} on startup err {}", app_id, e); - Some(AppLocalStatusCode::StartFailed) - } - } + self.push_front_cmd(&vec![(AppCmd::start(self.owner.clone(), app_id.clone()), 0)]).await; + None }, AppLocalStatusCode::Installing => { info!("find app {} status {} on startup, try install again", app_id, status_code); let version = status_a.lock().unwrap().version().map(|s|s.to_owned()); if let Some(version) = version { - match self.cmd_executor.as_ref().unwrap().execute_install( - status_a.clone(), - &AppCmd::install(self.owner.clone(), app_id.clone(), &version, true), - 0).await { - Ok(_) => { - None - } - Err(e) => { - error!("install app {} on startup err {}", app_id, e); - Some(AppLocalStatusCode::InstallFailed) - } - } + self.push_front_cmd(&vec![(AppCmd::install(self.owner.clone(), app_id.clone(), &version, true), 0)]).await; + None } else { Some(AppLocalStatusCode::InstallFailed) } }, AppLocalStatusCode::Uninstalling => { info!("find app {} status {} on startup, try uninstall again", app_id, status_code); - match self.cmd_executor.as_ref().unwrap().execute_uninstall( - status_a.clone(), - &AppCmd::uninstall(self.owner.clone(), app_id.clone()), - 0).await { - Ok(_) => { - None - } - Err(e) => { - error!("uninstall app {} on startup err {}", app_id, e); - Some(AppLocalStatusCode::UninstallFailed) - } - } + self.push_front_cmd(&vec![(AppCmd::uninstall(self.owner.clone(), app_id.clone()), 0)]).await; + None }, _ => None, }; diff --git a/src/service/app-manager/src/dapp.rs b/src/service/app-manager/src/dapp.rs index 4e7b6a165..66c838215 100644 --- a/src/service/app-manager/src/dapp.rs +++ b/src/service/app-manager/src/dapp.rs @@ -1,5 +1,5 @@ use cyfs_base::{BuckyError, BuckyErrorCode, BuckyResult}; -use cyfs_util::{get_app_dir, ProcessUtil}; +use cyfs_util::{get_app_dir}; use log::*; use serde::Deserialize; use serde_json::Value; @@ -7,9 +7,11 @@ use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::{Child, Command, ExitStatus}; +use std::str::FromStr; use std::sync::Mutex; use std::time::Duration; use wait_timeout::ChildExt; +use crate::process_util::{run, try_stop_process_by_pid}; const STATUS_CMD_TIME_OUT_IN_SECS: u64 = 15; const STOP_CMD_TIME_OUT_IN_SECS: u64 = 60; @@ -185,44 +187,6 @@ impl DApp { }) } - fn run(cmd: &str, dir: &Path, detach: bool, stdout: Option) -> BuckyResult { - let args: Vec<&str> = ProcessUtil::parse_cmd(cmd); - if args.len() == 0 { - error!("parse cmd {} failed, cmd empty?", cmd); - return Err(BuckyError::from(BuckyErrorCode::InvalidData)); - } - info!("run cmd {} in {}", cmd, dir.display()); - let program = which::which(args[0]).unwrap_or_else(|_| dir.join(args[0])); - info!("program full path: {}", program.display()); - let mut command = Command::new(program); - command.args(&args[1..]).current_dir(dir); - if let Some(out) = stdout { - command.stdout(out); - } - #[cfg(target_os = "windows")] - { - use std::os::windows::process::CommandExt; - command.creation_flags(0x08000000); - } - - if detach { - ProcessUtil::detach(&mut command); - } - - match command.spawn() { - Ok(p) => Ok(p), - Err(e) => { - error!( - "spawn app failed! cmd {}, dir {}, err {}", - cmd, - dir.display(), - e - ); - Err(BuckyError::from(BuckyErrorCode::ExecuteError)) - } - } - } - pub fn get_start_cmd(&self) -> String { self.info.start.clone() } @@ -231,39 +195,19 @@ impl DApp { Ok(self.info.executable.clone()) } - fn get_pid_file_path(&self) -> BuckyResult { - let pid_file = cyfs_util::get_cyfs_root_path() + fn get_pid_file_path(&self) -> PathBuf { + cyfs_util::get_cyfs_root_path() .join("run") - .join(format!("app_manager_app_{}", self.dec_id)); - Ok(pid_file) - } - - fn get_pid(&self) -> BuckyResult { - let lock_file = self.get_pid_file_path()?; - if !lock_file.is_file() { - return Err(BuckyError::new(BuckyErrorCode::NotFound, "no pid file")); - } - let result = std::fs::read_to_string(lock_file).unwrap(); - Ok(result) + .join(format!("app_manager_app_{}", self.dec_id)) } pub fn start(&self) -> BuckyResult { if !self.status()? { - let child = DApp::run(&self.info.start, &self.work_dir, true, None)?; - let id = child.id(); + let child = run(&self.info.start, &self.work_dir, true, None, Some(self.get_pid_file_path().as_path()))?; *self.process.lock().unwrap() = Some(child); - // mark pid - let lock_file = self.get_pid_file_path()?; - let buf = format!("{}", id).into_bytes(); - std::fs::write(lock_file, &buf).map_err(|e| { - let msg = format!("app[{}]{} write lock file failed! err {}", - self.dec_id, self.info.id, e); - error!("{}", &msg); - BuckyError::new(BuckyErrorCode::ExecuteError, msg) - })?; info!( - "start app:{} {} success! and write pid {:?}", - self.dec_id, self.info.id, id + "start app:{} {} success!", + self.dec_id, self.info.id ); return Ok(true); @@ -275,12 +219,12 @@ impl DApp { fn run_cmd( &self, cmd: &str, - dir: &Path, detach: bool, stdout: Option, time_out: u64, + record_pid: Option<&Path> ) -> BuckyResult { - let mut process = DApp::run(cmd, dir, detach, stdout)?; + let mut process = run(cmd, &self.work_dir, detach, stdout, record_pid)?; let app_id = self.info.id.as_str(); @@ -346,10 +290,10 @@ impl DApp { // 通过命令行判定app运行状态 let exit_code = self.run_cmd( &self.info.status, - &self.work_dir, false, None, STATUS_CMD_TIME_OUT_IN_SECS, + None, )?; Ok(exit_code != 0) } @@ -435,10 +379,10 @@ impl DApp { match self.run_cmd( &self.info.stop, - &self.work_dir, false, None, STOP_CMD_TIME_OUT_IN_SECS, + None, ) { Ok(code) => { if code != 0 { @@ -464,46 +408,7 @@ impl DApp { // system kill app by pid // appmanager 通过start记录的pid去兜底删除应用 fn _force_stop(&self) -> BuckyResult<()> { - let pid = self.get_pid(); - if pid.is_err() { - return Ok(()); - } - - let pid = pid.unwrap(); - info!( - "stop app[{}] by inner cmd failed, try to force kill by pid {}", - &self.dec_id, pid - ); - #[cfg(windows)] - { - let mut child = Command::new("taskkill") - .arg("/F") - .arg("/T") - .arg("/PID") - .arg(&pid) - .spawn() - .map_err(|e| { - error!("kill app:{} failed! err {}", pid, e); - BuckyError::from(BuckyErrorCode::ExecuteError) - })?; - let _ = child.wait(); - } - #[cfg(not(windows))] - { - let mut child = Command::new("kill") - .arg("-9") - .arg(&pid) - .spawn() - .map_err(|e| { - error!("kill app:{} failed! err {}", pid, e); - BuckyError::from(BuckyErrorCode::ExecuteError) - })?; - let _ = child.wait(); - } - - let lock_file = self.get_pid_file_path()?; - let _ = std::fs::remove_file(lock_file); - Ok(()) + try_stop_process_by_pid(self.get_pid_file_path().as_path(), Some(self.work_dir.as_path())) } // 这里做DecApp被安装后，执行前，根据配置文件需要做的预配置 @@ -516,10 +421,10 @@ impl DApp { // 就算执行不成功，也可以让开发者打包的时候就设置好，这里不成功不算错 let _ = self.run_cmd( &cmd, - &self.work_dir, false, None, 0, + None ); } } @@ -530,17 +435,17 @@ impl DApp { self.info.install.clone() } - pub fn install(&self) -> BuckyResult { + pub fn install(&self, pid_path: Option<&Path>) -> BuckyResult { let mut cmd_index = 0; for cmd in &self.info.install { let log_file = self.work_dir.join(format!("install_{}.log", cmd_index)); match self.run_cmd( cmd, - &self.work_dir, false, File::create(log_file).ok(), INSTALL_CMD_TIME_OUT_IN_SECS, + pid_path ) { Err(e) => { error!("run app:{} install cmd {} err {}", &self.info.id, cmd, e); diff --git a/src/service/app-manager/src/docker_api.rs b/src/service/app-manager/src/docker_api.rs index 905775ae3..c691641c5 100644 --- a/src/service/app-manager/src/docker_api.rs +++ b/src/service/app-manager/src/docker_api.rs @@ -89,6 +89,40 @@ fn run_docker_only_status>(args: Vec) -> BuckyResult BuckyResult { + let output = run_docker(vec!["container", "inspect", name, "--format", "{{.State.Status}}"])?.wait_with_output()?; + if output.status.success() { + let status = String::from_utf8(output.stdout).unwrap(); + Ok(status.trim() == "running") + } else { + info!("container inspect return error, tract as not running"); + Ok(false) + } +} + +pub(crate) fn stop_docker(name: &str) -> BuckyResult<()> { + info!("try to stop container[{}]", name); + if !(is_docker_running(name)?) { + info!("container {} not running", name); + return Ok(()); + } + + let args = vec!["stop", "-t", "30", &name]; + + let output = run_docker_only_status(args).map_err(|e| { + error!("stop container {} err {}", name, e); + e + })?; + + if output.success() { + info!("stop container {} success", name); + Ok(()) + } else { + error!("stop container {} fail, exit code {}", name, output.code().map(|i|{i.to_string()}).unwrap_or("signal".to_owned())); + Err(BuckyError::from(BuckyErrorCode::Failed)) + } +} + fn add_bind_volume, Q: AsRef>(args: &mut Vec, source: P, target: Q, read_only: bool) { args.push("-v".to_owned()); let mounts = if read_only { @@ -218,6 +252,10 @@ impl DockerApi { // 构造 start.sh let start_sh_path = dockerfile_dir.join("start.sh"); + if start_sh_path.is_dir() { + warn!("remove wrong sh path dir {}", start_sh_path.display()); + let _ = std::fs::remove_dir_all(&start_sh_path); + } let gateway_ip = DockerApi::get_network_gateway_ip()?; info!("get docker start shell 's gateway ip: {}", gateway_ip); @@ -226,6 +264,7 @@ impl DockerApi { .replace("{gateway_ip}", &gateway_ip); info!("docker start shell content: {}", &shell); std::fs::write(&start_sh_path, shell)?; + info!("write start shell file {}", start_sh_path.display()); Ok(()) } @@ -236,7 +275,6 @@ impl DockerApi { install_cmd: Vec, ) -> BuckyResult<()> { self.prepare(id)?; - // TODO 在容器内执行install命令 for cmd in install_cmd { info!("start app {} install cmd {} in docker", id, cmd); let container_name = format!("decapp-{}-install", id.to_lowercase()); @@ -267,6 +305,7 @@ impl DockerApi { None => { let msg = format!("app {} run install cmd {} not return after {} secs, kill", id, &cmd, INSTALL_CMD_TIME_OUT_IN_SECS); error!("{}", &msg); + stop_docker(&container_name)?; let _ = child.kill(); let _ = child.wait(); return Err(BuckyError::new(BuckyErrorCode::Timeout, msg)); @@ -419,38 +458,12 @@ impl DockerApi { pub fn stop(&self, id: &str) -> BuckyResult<()> { let container_name = format!("decapp-{}", id.to_lowercase()); - info!("try to stop container[{}]", container_name); - if !(self.is_running(id)?) { - info!("container {} not running", container_name); - return Ok(()); - } - - let args = vec!["stop", "-t", "30", &container_name]; - - let output = run_docker_only_status(args).map_err(|e| { - error!("stop container {} err {}", container_name, e); - e - })?; - - if output.success() { - info!("stop container {} success", container_name); - Ok(()) - } else { - error!("stop container {} fail, exit code {}", container_name, output.code().map(|i|{i.to_string()}).unwrap_or("signal".to_owned())); - Err(BuckyError::from(BuckyErrorCode::Failed)) - } + stop_docker(&container_name) } pub fn is_running(&self, id: &str) -> BuckyResult { let container_name = format!("decapp-{}", id.to_lowercase()); - let output = run_docker(vec!["container", "inspect", &container_name, "--format", "{{.State.Status}}"])?.wait_with_output()?; - if output.status.success() { - let status = String::from_utf8(output.stdout).unwrap(); - Ok(status.trim() == "running") - } else { - info!("container inspect return error, tract as not running"); - Ok(false) - } + is_docker_running(&container_name) } } diff --git a/src/service/app-manager/src/lib.rs b/src/service/app-manager/src/lib.rs index 179e9e253..43252b7ae 100644 --- a/src/service/app-manager/src/lib.rs +++ b/src/service/app-manager/src/lib.rs @@ -1,2 +1,3 @@ +mod process_util; pub mod package; pub mod dapp; \ No newline at end of file diff --git a/src/service/app-manager/src/main.rs b/src/service/app-manager/src/main.rs index ab042bd34..7e1db4fb5 100644 --- a/src/service/app-manager/src/main.rs +++ b/src/service/app-manager/src/main.rs @@ -5,7 +5,7 @@ use app_manager_lib::{AppManagerConfig}; use clap::{App}; use cyfs_base::*; use cyfs_core::DecAppId; -use cyfs_lib::SharedCyfsStack; +use cyfs_lib::{SharedCyfsStack, UtilGetDeviceStaticInfoRequest}; use cyfs_util::process::{ prepare_args, set_process_cmd_funcs, ProcessAction, ProcessCmdFuncs, }; @@ -25,6 +25,7 @@ mod docker_network_manager; mod event_handler; mod non_helper; mod package; +mod process_util; struct AppManagerProcessFuncs { config: AppManagerConfig, @@ -65,7 +66,7 @@ impl ProcessCmdFuncs for AppManagerProcessFuncs { } } -async fn main_run() { +async fn main_run() -> BuckyResult<()> { //cyfs_base::init_log_with_isolate_bdt(APP_MANAGER_NAME, Some("debug"), None); //let action = cyfs_util::process::check_cmd_and_exec(APP_MANAGER_NAME); @@ -123,6 +124,12 @@ async fn main_run() { } let _ = cyfs_stack.wait_online(None).await; + let status = cyfs_stack.util().get_device_static_info(UtilGetDeviceStaticInfoRequest::new()).await?; + if !status.info.zone_role.is_active_ood() { + warn!("app manager started but not active ood! zone role {}", status.info.zone_role); + std::process::exit(1); + } + let mut app_manager = AppManagerEx::new(cyfs_stack, app_config); if let Err(e) = app_manager.init().await { @@ -133,6 +140,8 @@ async fn main_run() { AppManagerEx::start(Arc::new(app_manager)).await; async_std::task::block_on(async_std::future::pending::<()>()); + + Ok(()) } /*fn kill_service_start_process(service_name: &str) -> bool { @@ -159,7 +168,7 @@ async fn main_run() { return false; }*/ -fn main() { +fn main() -> BuckyResult<()> { cyfs_debug::ProcessDeadHelper::patch_task_min_thread(); async_std::task::block_on(main_run()) diff --git a/src/service/app-manager/src/non_helper.rs b/src/service/app-manager/src/non_helper.rs index a9bfa3115..ca8338965 100644 --- a/src/service/app-manager/src/non_helper.rs +++ b/src/service/app-manager/src/non_helper.rs @@ -438,7 +438,7 @@ impl NonHelper { .root_state_stub(None, None) .create_path_op_env() .await?; - op_env.lock(vec![path.to_owned()], 0).await.unwrap(); + op_env.lock(vec![path.to_owned()], 0).await?; op_env.set_with_path(path, obj_id, None, true).await?; op_env.commit().await?; diff --git a/src/service/app-manager/src/process_util.rs b/src/service/app-manager/src/process_util.rs new file mode 100644 index 000000000..cfaad9583 --- /dev/null +++ b/src/service/app-manager/src/process_util.rs @@ -0,0 +1,94 @@ +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::str::FromStr; +use sysinfo::{Pid, ProcessExt, ProcessRefreshKind, RefreshKind, SystemExt}; +use cyfs_base::{BuckyError, BuckyErrorCode, BuckyResult}; +use log::*; +use cyfs_core::DecAppId; +use cyfs_util::ProcessUtil; + +pub fn run(cmd: &str, work_dir: &Path, detach: bool, stdout: Option, record_pid: Option<&Path>) -> BuckyResult { + let args: Vec<&str> = ProcessUtil::parse_cmd(cmd); + if args.len() == 0 { + error!("parse cmd {} failed, cmd empty?", cmd); + return Err(BuckyError::from(BuckyErrorCode::InvalidData)); + } + info!("run cmd {} in {}", cmd, work_dir.display()); + let program = which::which(args[0]).unwrap_or_else(|_| work_dir.join(args[0])); + info!("program full path: {}", program.display()); + let mut command = Command::new(program); + command.args(&args[1..]).current_dir(work_dir); + if let Some(out) = stdout { + command.stdout(out); + } + #[cfg(target_os = "windows")] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x08000000); + } + + if detach { + ProcessUtil::detach(&mut command); + } + + match command.spawn() { + Ok(p) => { + if let Some(path) = record_pid { + info!("write process pid {} to {}", p.id(), path.display()); + let _ = std::fs::write(path, p.id().to_string().as_bytes()); + } + Ok(p) + }, + Err(e) => { + error!("spawn app failed! cmd {}, dir {}, err {}", cmd, work_dir.display(), e); + Err(BuckyError::from(BuckyErrorCode::ExecuteError)) + } + } +} + +pub fn try_stop_process_by_pid(pid_path: &Path, match_work_dir: Option<&Path>) -> BuckyResult<()> { + if pid_path.is_file() { + let pid = std::fs::read_to_string(pid_path)?; + let info = sysinfo::System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::new())); + if let Some(process) = info.process(Pid::from_str(&pid)?) { + if let Some(path) = match_work_dir { + if process.cwd() != path { + warn!("pid {} work dir mismatch! except {}, actual {}. not kill", &pid, path.display(), process.cwd().display()); + return Ok(()); + } + } + info!("try to force kill process by pid {}", pid); + let cmd; + #[cfg(windows)] + { + cmd = format!("taskkill /F /T /PID {}", &pid); + } + #[cfg(not(windows))] + { + cmd = format!("kill -9 {}", &pid); + } + run(&cmd, &Path::new("."), false, None, None)?.wait()?; + } + } else { + info!("not found or not file: pid path {}", pid_path.display()); + } + + if pid_path.is_dir() { + if let Ok(_) = std::fs::remove_dir_all(pid_path) { + info!("delete pid path {}?", pid_path.display()); + } + } else { + if let Ok(_) = std::fs::remove_file(pid_path) { + info!("delete pid file {}", pid_path.display()); + } + } + + Ok(()) +} + +pub fn get_install_pid_file_path(id: &DecAppId) -> PathBuf { + cyfs_util::get_cyfs_root_path() + .join("run") + .join(format!("app_manager_app_install_{}", id)) +} \ No newline at end of file diff --git a/src/service/chunk-manager/src/chunk_store.rs b/src/service/chunk-manager/src/chunk_store.rs index f7fa79682..d0cd7f02a 100644 --- a/src/service/chunk-manager/src/chunk_store.rs +++ b/src/service/chunk-manager/src/chunk_store.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::io::{Error}; +use std::io::{Error, ErrorKind}; use cyfs_base::{BuckyResult, ChunkId}; @@ -12,6 +12,7 @@ use async_std::sync::Mutex; use async_std::sync::RwLock; use std::io::prelude::*; +use crate::chunk_processor::set_chunk; #[derive(Debug)] @@ -85,6 +86,14 @@ impl ChunkStore { chunk_lock.read().await; let chunk_path = self.chunk_dir.join(chunk_id.to_string()); + let file_len = std::fs::metadata(&chunk_path)?.len(); + let chunk_len = chunk_id.len(); + if chunk_len as u64 != file_len { + error!("file {} len mismatch!, except {} actual {}", chunk_path.display(), chunk_len, file_len); + info!("delete chunk file {}", chunk_path.display()); + let _ = std::fs::remove_file(chunk_path); + return Err(Error::from(ErrorKind::NotFound)); + } let file = File::open(chunk_path.as_path()).await?; let reader = BufReader::new(file); Ok(reader) @@ -104,16 +113,17 @@ impl ChunkStore { chunk_lock.write().await; - info!("[set_chunk], start write chunk to file"); let chunk_path = self.chunk_dir.join(chunk_id.to_string()); - info!("[set_chunk], create chunk file {}", chunk_path.display()); - // let mut file = File::create(chunk_path.as_path()).await?; - let mut file = std::fs::File::create(&chunk_path)?; - - info!("[set_chunk], write chunk file {}", chunk_path.display()); - // file.write(chunk).await?; - file.write_all(chunk)?; + info!("[set_chunk], write chunk {} to {}", chunk_id, chunk_path.display()); + if let Err(e) = std::fs::write(&chunk_path, chunk) { + error!("set chunk {} err {}", chunk_path.display(), e); + if chunk_path.exists() { + info!("delete chunk file {}", chunk_path.display()); + std::fs::remove_file(&chunk_path)?; + } + return Err(e) + } info!("[set_chunk], end write chunk file {}", chunk_path.display()); Ok(()) diff --git a/src/service/ood-control/Cargo.toml b/src/service/ood-control/Cargo.toml index d6520b3f1..f2caf3dca 100644 --- a/src/service/ood-control/Cargo.toml +++ b/src/service/ood-control/Cargo.toml @@ -13,6 +13,7 @@ cyfs-base = { path = "../../component/cyfs-base" } cyfs-debug = { path = "../../component/cyfs-debug" } cyfs-util = { path = "../../component/cyfs-util" } cyfs-lib = { path = "../../component/cyfs-lib" } +cyfs-backup = { path = "../../component/cyfs-backup" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" log = "0.4" @@ -38,3 +39,6 @@ android_logger = "0.11" [lib] name = "ood_control" crate-type = ["lib", "cdylib", "staticlib"] + +[dev-dependencies] +surf = { version = '2.3', default-features = false, features = ['h1-client-rustls'] } \ No newline at end of file diff --git a/src/service/ood-control/src/controller.rs b/src/service/ood-control/src/controller.rs index fcd344956..2ed66cda7 100644 --- a/src/service/ood-control/src/controller.rs +++ b/src/service/ood-control/src/controller.rs @@ -2,6 +2,7 @@ use super::bind::*; use super::device_info::*; use super::request::*; use cyfs_base::*; +use super::restore::RestoreController; use once_cell::sync::OnceCell; use std::collections::hash_map::{Entry, HashMap}; @@ -19,7 +20,7 @@ pub(crate) struct ControllerImpl { ext_info_file: PathBuf, zone_owner_desc_file: PathBuf, - // 设备信息 + // Device info device_info: DeviceInfo, bind_state: BindState, @@ -29,6 +30,8 @@ pub(crate) struct ControllerImpl { access_info: OnceCell, external_servers: Mutex>>, + + restore_controller: RestoreController, } impl ControllerImpl { @@ -39,7 +42,7 @@ impl ControllerImpl { let bind_state = BindState::new(desc_file.clone(), sec_file.clone(), ext_info_file.clone()); let _r = bind_state.load(); - // 如果尚未绑定，那么开启一个检测 + // Start the check routine if not bind yet if !bind_state.is_bind() { bind_state.start_monitor_bind(); } @@ -60,6 +63,8 @@ impl ControllerImpl { access_info: OnceCell::new(), external_servers: Mutex::new(vec![]), + + restore_controller: RestoreController::new(), } } @@ -351,8 +356,12 @@ impl Controller { pub(crate) fn new() -> Self { Self(Arc::new(ControllerImpl::new())) } + + pub fn restore_controller(&self) -> &RestoreController { + &self.0.restore_controller + } - // 记录一次check请求 + // Record a check request once pub fn on_check_request(&self, source: &str) { self.0.on_check_request(source) } diff --git a/src/service/ood-control/src/http_server.rs b/src/service/ood-control/src/http_server.rs index f3ac0d6f2..fd9c2e492 100644 --- a/src/service/ood-control/src/http_server.rs +++ b/src/service/ood-control/src/http_server.rs @@ -12,6 +12,11 @@ enum RequestType { Check, Bind, SystemInfo, + + CreateRestoreTask, + GetRestoreTaskStatus, + GetRestoreTaskList, + AbortRestoreTask, } pub(crate) struct HandlerEndpoint { @@ -54,7 +59,7 @@ impl HandlerEndpoint { Ok(()) } - async fn process_request(&self, mut req: ::tide::Request) -> Response { + async fn process_request(&self, mut req: ::tide::Request<()>) -> Response { if self.access_token.is_some() { if let Err(e) = self.check_token(&req.url()) { return cyfs_lib::RequestorHelper::trans_error(e); @@ -78,6 +83,30 @@ impl HandlerEndpoint { } }, RequestType::SystemInfo => self.on_get_system_info_request().await, + + RequestType::CreateRestoreTask => { + self.handler + .restore_controller() + .process_create_remote_restore_task_request(req) + .await + } + + RequestType::GetRestoreTaskStatus => self + .handler + .restore_controller() + .process_get_remote_restore_task_status_request(req), + + RequestType::AbortRestoreTask => { + self.handler + .restore_controller() + .process_abort_remote_restore_task_request(req) + .await + } + + RequestType::GetRestoreTaskList => self + .handler + .restore_controller() + .process_get_remote_restore_task_list_request(req), } } @@ -159,6 +188,60 @@ impl HandlerEndpoint { handler.to_owned(), )); + //// restore related services + + // start restore task + server.at("/restore").post(HandlerEndpoint::new( + RequestType::CreateRestoreTask, + access_token.clone(), + handler.to_owned(), + )); + + server.at("/restore/").post(HandlerEndpoint::new( + RequestType::CreateRestoreTask, + access_token.clone(), + handler.to_owned(), + )); + + // get task status + server.at("/restore/:task_id").get(HandlerEndpoint::new( + RequestType::GetRestoreTaskStatus, + access_token.clone(), + handler.to_owned(), + )); + + server.at("/restore/:task_id/").get(HandlerEndpoint::new( + RequestType::GetRestoreTaskStatus, + access_token.clone(), + handler.to_owned(), + )); + + // abort task + server.at("/restore/:task_id").delete(HandlerEndpoint::new( + RequestType::AbortRestoreTask, + access_token.clone(), + handler.to_owned(), + )); + + server.at("/restore/:task_id/").delete(HandlerEndpoint::new( + RequestType::AbortRestoreTask, + access_token.clone(), + handler.to_owned(), + )); + + // get task list + server.at("/restore/tasks").get(HandlerEndpoint::new( + RequestType::GetRestoreTaskList, + access_token.clone(), + handler.to_owned(), + )); + + server.at("/restore/tasks/").get(HandlerEndpoint::new( + RequestType::GetRestoreTaskList, + access_token.clone(), + handler.to_owned(), + )); + // external let all = handler.fetch_all_external_servers(); for item in all { @@ -168,12 +251,9 @@ impl HandlerEndpoint { } #[async_trait] -impl tide::Endpoint for HandlerEndpoint -where - State: Clone + Send + Sync + 'static, -{ - async fn call(&self, req: ::tide::Request) -> ::tide::Result { +impl tide::Endpoint<()> for HandlerEndpoint { + async fn call(&self, req: ::tide::Request<()>) -> ::tide::Result { let resp = self.process_request(req).await; Ok(resp) } -} \ No newline at end of file +} diff --git a/src/service/ood-control/src/lib.rs b/src/service/ood-control/src/lib.rs index 88d7d4281..84abc719b 100644 --- a/src/service/ood-control/src/lib.rs +++ b/src/service/ood-control/src/lib.rs @@ -7,6 +7,7 @@ mod http_server; pub mod interface; mod request; mod ood_controller; +mod restore; pub use app_bind_manager::AppBindManager; pub use controller::*; @@ -16,6 +17,9 @@ pub use interface::{ pub use request::*; pub use ood_controller::*; +#[cfg(test)] +mod test; + #[macro_use] extern crate log; diff --git a/src/service/ood-control/src/restore.rs b/src/service/ood-control/src/restore.rs new file mode 100644 index 000000000..5f94219d4 --- /dev/null +++ b/src/service/ood-control/src/restore.rs @@ -0,0 +1,125 @@ +use cyfs_backup::*; +use cyfs_base::*; +use cyfs_lib::RequestorHelper; + +use tide::{Request, Response, StatusCode}; + +pub struct RestoreController { + restore_manager: RemoteRestoreManager, +} + +impl RestoreController { + pub fn new() -> Self { + Self { + restore_manager: RemoteRestoreManager::new(), + } + } + + pub async fn process_create_remote_restore_task_request( + &self, + mut req: Request<()>, + ) -> tide::Response { + match req.body_json().await { + Ok(param) => match self.start_remote_restore(param) { + Ok(()) => RequestorHelper::new_ok_response(), + Err(e) => RequestorHelper::trans_error(e), + }, + Err(e) => { + let msg = format!("parse restore params error: {}", e); + error!("{}", msg); + + Response::builder(StatusCode::BadRequest).body(msg).build() + } + } + } + + fn start_remote_restore(&self, params: RemoteRestoreParams) -> BuckyResult<()> { + // In terms of ood-control, there is currently one and only one restore task in existence! + let list = self.restore_manager.get_tasks(); + if !list.is_empty() { + let msg = format!("restore task already exists: {:?}", list); + error!("{}", msg); + + return Err(BuckyError::new(BuckyErrorCode::AlreadyExists, msg)); + } + + self.restore_manager.start_remote_restore(params) + } + + pub fn process_get_remote_restore_task_status_request( + &self, + req: Request<()>, + ) -> tide::Response { + match self.get_task_status(req) { + Ok(status) => { + let mut resp: Response = RequestorHelper::new_ok_response(); + let body = serde_json::to_string(&status).unwrap(); + + resp.set_content_type(tide::http::mime::JSON); + resp.set_body(body); + + resp + } + Err(e) => { + RequestorHelper::trans_error(e) + } + } + } + + fn get_task_status(&self, req: Request<()>) -> BuckyResult { + let task_id = req.param("task_id").map_err(|e| { + let msg = format!("invalid task_id segment: {}", e); + error!("{}", msg); + + BuckyError::new(BuckyErrorCode::InvalidFormat, msg) + })?; + + self.restore_manager.get_task_status(task_id) + } + + pub async fn process_abort_remote_restore_task_request( + &self, + req: Request<()>, + ) -> tide::Response { + match self.abort_task(req).await { + Ok(status) => { + let mut resp: Response = RequestorHelper::new_ok_response(); + let body = serde_json::to_string(&status).unwrap(); + + resp.set_content_type(tide::http::mime::JSON); + resp.set_body(body); + + resp + } + Err(e) => { + RequestorHelper::trans_error(e) + } + } + } + + async fn abort_task(&self, req: Request<()>) -> BuckyResult<()> { + let task_id = req.param("task_id").map_err(|e| { + let msg = format!("invalid task_id segment: {}", e); + error!("{}", msg); + + BuckyError::new(BuckyErrorCode::InvalidFormat, msg) + })?; + + self.restore_manager.abort_task(task_id).await + } + + pub fn process_get_remote_restore_task_list_request( + &self, + _req: Request<()>, + ) -> tide::Response { + let list = self.restore_manager.get_tasks(); + + let mut resp: Response = RequestorHelper::new_ok_response(); + let body = serde_json::to_string(&list).unwrap(); + + resp.set_content_type(tide::http::mime::JSON); + resp.set_body(body); + + resp + } +} diff --git a/src/service/ood-control/src/test.rs b/src/service/ood-control/src/test.rs new file mode 100644 index 000000000..8e1190b96 --- /dev/null +++ b/src/service/ood-control/src/test.rs @@ -0,0 +1,63 @@ +use crate::*; +use cyfs_backup::*; + + +async fn start() { + let param = ControlInterfaceParam { + mode: OODControlMode::Runtime, + tcp_port: None, + + // if device binded already，should not bind public address to avoid risk + require_access_token: false, + tcp_host: None, + addr_type: ControlInterfaceAddrType::V4, + }; + + let control_interface = ControlInterface::new(param, &OOD_CONTROLLER); + control_interface.start().await.unwrap(); +} + +async fn test_restore() { + start().await; + + let control_url = "http://127.0.0.1:1321/restore"; + let param = RemoteRestoreParams::new("restore_task_1", "http://127.0.0.1:8887/test65/${filename}?token=123456"); + + let resp = surf::post(control_url).body_json(¶m).unwrap().await.unwrap(); + assert!(resp.status().is_success()); + + let tasks_url = format!("{}/tasks", control_url); + let mut resp = surf::get(&tasks_url).await.unwrap(); + assert!(resp.status().is_success()); + let list = resp.body_string().await.unwrap(); + info!("task list: {}", list); + + let mut count = 0; + let task_url = format!("{}/{}", control_url, param.id); + loop { + let mut resp = surf::get(&task_url).await.unwrap(); + assert!(resp.status().is_success()); + + let status = resp.body_string().await.unwrap(); + info!("status: {}", status); + + async_std::task::sleep(std::time::Duration::from_secs(5)).await; + + count += 1; + if count >= 5 { + break; + } + } + + let resp = surf::delete(&task_url).await.unwrap(); + assert!(resp.status().is_success()); + + let resp = surf::get(&task_url).await.unwrap(); + assert_eq!(resp.status(), http_types::StatusCode::NotFound); +} + +#[test] +fn test() { + cyfs_base::init_simple_log("test-ood-control", None); + async_std::task::block_on(test_restore()); +} \ No newline at end of file diff --git a/src/service/ood-daemon/src/service/service.rs b/src/service/ood-daemon/src/service/service.rs index e515bf1ef..4b5931a77 100644 --- a/src/service/ood-daemon/src/service/service.rs +++ b/src/service/ood-daemon/src/service/service.rs @@ -147,8 +147,9 @@ impl Service { return; } - // 如果进程是我们自己拉起的，那么优先尝试使用进程对象来停止 - if self.state.lock().unwrap().process.is_some() { + // If the process is launched by ourselves, then give priority to trying the process object to stop + let process_exists = self.state.lock().unwrap().process.is_some(); + if process_exists { self.kill_process(); } @@ -265,22 +266,27 @@ impl Service { ); match cmd.spawn() { - Ok(mut child) => match child.wait() { - Ok(status) => { - info!( - "stop cmd complete, service={}, code={}, cmd={}", - self.name, - status.code().unwrap_or_default(), - v, - ); - } - Err(err) => { - error!( - "wait stop cmd error, service={}, err={}, cmd={}", - self.name, err, v - ); + Ok(mut child) => { + let pid = child.id(); + + match child.wait() { + Ok(status) => { + info!( + "stop cmd complete, service={}, code={}, cmd={}, pid={}", + self.name, + status.code().unwrap_or_default(), + v, + pid, + ); + } + Err(err) => { + error!( + "wait stop cmd error, service={}, err={}, cmd={}, pid={}", + self.name, err, v, pid, + ); + } } - }, + } Err(err) => { error!( "exec stop cmd error! service={}, err={}, cmd={}", @@ -322,7 +328,12 @@ impl Service { let mut cmd = self.gen_cmd(&v, true, ignore_output).unwrap(); match cmd.spawn() { Ok(p) => { - info!("spawn service success, service={}, cmd={}", self.name, v); + info!( + "spawn service success, service={}, cmd={}, pid={}", + self.name, + v, + p.id() + ); // 保存process句柄 self.state.lock().unwrap().process = Some(p); @@ -350,36 +361,46 @@ impl Service { need_cmd_check = true; } else { let mut process = process.unwrap(); + let pid = process.id(); match process.try_wait() { Ok(Some(status)) => { info!( - "service exited, name={}, status={}, current state={}", - self.name, status, state.state + "service exited, name={}, status={}, current state={}, pid={}", + self.name, status, state.state, pid, ); match process.wait() { Ok(_) => { - info!("wait service process complete! name={}", self.name); + info!( + "wait service process complete! name={}, pid={}", + self.name, pid + ); } Err(e) => { - info!("wait service process error! name={}, err={}", self.name, e); + info!( + "wait service process error! name={}, pid={}, err={}", + self.name, pid, e + ); } } state.state = ServiceState::Stop; } Ok(None) => { - debug!("service still running: {}", self.name); + debug!("service still running: name={}, pid={}", self.name, pid); // 仍然在运行，需要保留进程object并继续等待 state.process = Some(process); if state.state != ServiceState::Run { - warn!("process object exists and still running but state is not run! service={}, current state={}", self.name, state.state); + warn!("process object exists and still running but state is not run! name={}, current state={}, pid={}", self.name, state.state, pid); state.state = ServiceState::Run; } } Err(e) => { // 出错的情况下，直接放弃这个进程object，并使用cmd进一步检测 - error!("update service state error, name={}, err={}", self.name, e); + error!( + "update service state error, name={}, err={}, pid={}", + self.name, e, pid + ); need_cmd_check = true; } } @@ -413,22 +434,26 @@ impl Service { cmd.args(&["--fid", &self.fid]); match cmd.spawn() { - Ok(mut child) => match child.wait() { - Ok(status) => { - exit_code = status.code().unwrap_or_default(); + Ok(mut child) => { + let pid = child.id(); - debug!( - "check cmd complete, name={}, code={}, cmd={}", - self.name, exit_code, v - ); - } - Err(err) => { - error!( - "wait check cmd error, name={}, err={}, cmd={}", - self.name, err, v - ); + match child.wait() { + Ok(status) => { + exit_code = status.code().unwrap_or_default(); + + debug!( + "check cmd complete, name={}, code={}, cmd={}, pid={}", + self.name, exit_code, v, pid, + ); + } + Err(err) => { + error!( + "wait check cmd error, name={}, err={}, cmd={}, pid={}", + self.name, err, v, pid, + ); + } } - }, + } Err(err) => { error!( "exec check cmd error! name={}, err={}, cmd={}", @@ -565,16 +590,20 @@ impl Service { self.name, self.full_fid, ); - // 拷贝文件前，确保服务已经停止 + // Before copying the file, make sure that the service has stopped self.sync_state(ServiceState::Stop); - // 加载新的包文件 - let ret = self.info.lock().unwrap().load_package_file(&file).map_err(|e| { - self.info.lock().unwrap().state = ServicePackageLocalState::Invalid; - e - })?; + // Load new package files + let ret = { + let mut info = self.info.lock().unwrap(); + let ret = info.load_package_file(&file).map_err(|e| { + info.state = ServicePackageLocalState::Invalid; + e + })?; - self.info.lock().unwrap().state = ServicePackageLocalState::Ready; + info.state = ServicePackageLocalState::Ready; + ret + }; self.sync_state(ServiceState::Stop); diff --git a/src/tests/cyfs-stack-test/Cargo.toml b/src/tests/cyfs-stack-test/Cargo.toml index 522fcf5a1..4dbdd2ce8 100644 --- a/src/tests/cyfs-stack-test/Cargo.toml +++ b/src/tests/cyfs-stack-test/Cargo.toml @@ -34,4 +34,5 @@ serde_json = "1.0" async-recursion = "1.0" once_cell = "1.12" serde = { version = '1.0', features = ['derive'] } -chrono = '0.4' \ No newline at end of file +chrono = '0.4' +http-types = "2.12" \ No newline at end of file diff --git a/src/tests/cyfs-stack-test/src/case/acl_handler.rs b/src/tests/cyfs-stack-test/src/case/acl_handler.rs new file mode 100644 index 000000000..abc3402bd --- /dev/null +++ b/src/tests/cyfs-stack-test/src/case/acl_handler.rs @@ -0,0 +1,128 @@ +use cyfs_base::*; +use cyfs_lib::*; +use cyfs_util::*; +use zone_simulator::*; + +use http_types::Url; + +const USER_ACCESS_TOKEN: &str = "123456"; + +struct OnDynamicAcl; + +pub async fn test() { + test_handler().await; + + info!("test all acl handler case success!"); + + async_std::task::sleep(std::time::Duration::from_secs(1000)).await; +} + +#[async_trait::async_trait] +impl EventListenerAsyncRoutine for OnDynamicAcl { + async fn call(&self, param: &RouterHandlerAclRequest) -> BuckyResult { + info!("will handle dynamic acl: {}, query={:?}", param.request.req_path, param.request.req_query_string); + + let mut action = AclAction::Accept; + let querys: Vec<_> = param.request.req_query_string.as_ref().unwrap().split('&').collect(); + for query in querys { + if let Some((k, v)) = query.split_once('=') { + if k == "token" { + if v != USER_ACCESS_TOKEN { + warn!("invalid user token: {}", v); + action = AclAction::Reject; + } else { + action = AclAction::Accept; + } + + break; + } + } else { + unreachable!("should not come here in out test case! {}", query); + } + } + + // let url = Url::parse(¶m.request.req_path).unwrap(); + + let resp = AclHandlerResponse { + action, + }; + + let result = RouterHandlerAclResult { + action: RouterHandlerAction::Response, + request: None, + response: Some(Ok(resp)), + }; + + Ok(result) + } +} + +async fn test_handler() { + let device1 = TestLoader::get_shared_stack(DeviceIndex::User1Device1); + let device2 = TestLoader::get_shared_stack(DeviceIndex::User2Device1); + + const TEST_REQ_PATH: &str = "/test/dynamic/call"; + + // First set handler access for {req_path} for clean test environment + device1 + .root_state_meta_stub(None, None) + .clear_access() + .await + .unwrap(); + + // Add rmeta 'Handler' access for specific req path + let item = GlobalStatePathAccessItem { + path: TEST_REQ_PATH.to_owned(), + access: GlobalStatePathGroupAccess::Handler, + }; + + device1 + .root_state_meta_stub(None, None) + .add_access(item) + .await + .unwrap(); + + // Add acl handler to handle request + let handler = OnDynamicAcl {}; + device1.router_handlers().add_handler( + RouterHandlerChain::Acl, + "test-dynamic-acl-1", + 0, + Some("*".to_owned()), + Some(TEST_REQ_PATH.to_owned()), + RouterHandlerAction::Reject, + Some(Box::new(handler)), + ).unwrap(); + + async_std::task::sleep(std::time::Duration::from_secs(2)).await; + + // try get object from device1 from /test/dynamic/call as current dec identity + let target = device1.local_device_id().object_id().to_owned(); + let stub = device2.root_state_accessor_stub(Some(target), None); + + let full_req_path = format!("{}?token={}", TEST_REQ_PATH, USER_ACCESS_TOKEN); + + let resp: Result = stub.get_object_by_path(full_req_path).await; + match resp { + Err(e) => { + assert_eq!(e.code(), BuckyErrorCode::NotFound); + info!("get object from path: {:?}", e); + } + Ok(_) => { + unreachable!(); + } + } + + let full_req_path = format!("{}?token={}", TEST_REQ_PATH, "12345"); + + let resp: Result = stub.get_object_by_path(full_req_path).await; + match resp { + Err(e) => { + assert_eq!(e.code(), BuckyErrorCode::PermissionDenied); + info!("get object from path: {:?}", e); + } + Ok(_) => { + unreachable!(); + } + } +} diff --git a/src/tests/cyfs-stack-test/src/case/crypto.rs b/src/tests/cyfs-stack-test/src/case/crypto.rs index f4ae23eb0..2d3eef9c7 100644 --- a/src/tests/cyfs-stack-test/src/case/crypto.rs +++ b/src/tests/cyfs-stack-test/src/case/crypto.rs @@ -174,7 +174,7 @@ impl EventListenerAsyncRoutine action: RouterHandlerAction::Response, request: None, response: Some(Ok(AclHandlerResponse { - access: AclAccess::Accept, + action: AclAction::Accept, })), }; diff --git a/src/tests/cyfs-stack-test/src/case/mod.rs b/src/tests/cyfs-stack-test/src/case/mod.rs index d8c25c320..702eda46f 100644 --- a/src/tests/cyfs-stack-test/src/case/mod.rs +++ b/src/tests/cyfs-stack-test/src/case/mod.rs @@ -30,6 +30,7 @@ mod object_meta_access; mod shared_stack; mod context; mod backup; +mod acl_handler; pub async fn test_restart() { let stack = TestLoader::get_stack(DeviceIndex::User1OOD); @@ -39,7 +40,7 @@ pub async fn test_restart() { } pub async fn test() { - root_state::test().await; + acl_handler::test().await; return; backup::test().await; diff --git a/src/tests/cyfs-stack-test/src/main.rs b/src/tests/cyfs-stack-test/src/main.rs index d71aa91eb..749f31531 100644 --- a/src/tests/cyfs-stack-test/src/main.rs +++ b/src/tests/cyfs-stack-test/src/main.rs @@ -10,9 +10,9 @@ use cyfs_debug::*; async fn main_run() { CyfsLoggerBuilder::new_app("cyfs-stack-test") - .level("warn") - .console("warn") - .enable_bdt(Some("info"), Some("error")) + .level("info") + .console("info") + .enable_bdt(Some("info"), Some("warn")) .disable_file_config(true) .file(true) .build() diff --git a/src/tools/bdt-tool/Cargo.toml b/src/tools/bdt-tool/Cargo.toml new file mode 100644 index 000000000..85af04871 --- /dev/null +++ b/src/tools/bdt-tool/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "bdt-tool" +version = "0.1.0" +edition = "2021" +license = "BSD-2-Clause" +description = "Rust bdt-tool package" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cyfs-debug = { path = "../../component/cyfs-debug" } +cyfs-base = { path = "../../component/cyfs-base" } +cyfs-bdt = { path = "../../component/cyfs-bdt" } +cyfs-util = { path = '../../component/cyfs-util', version = '0.6' } +async-std = { version = "1.11", features = ["unstable", "attributes"] } +clap = "2.34.0" +log = "0.4" +md5 = "0.7.0" \ No newline at end of file diff --git a/src/tools/bdt-tool/src/main.rs b/src/tools/bdt-tool/src/main.rs new file mode 100644 index 000000000..4b4fd6d7e --- /dev/null +++ b/src/tools/bdt-tool/src/main.rs @@ -0,0 +1,356 @@ +use std::{ + io::{Read}, + path::Path, + str::FromStr, + time::Duration, + net::Shutdown, +}; +use clap::{ + App, + SubCommand, + Arg +}; +use async_std::{ + future, +}; +use cyfs_base::*; +use cyfs_bdt::*; +use log::*; + +fn load_dev_by_path(path: &str) -> Option { + let desc_path = Path::new(path); + if desc_path.exists() { + let mut file = std::fs::File::open(desc_path).unwrap(); + let mut buf = Vec::::new(); + let _ = file.read_to_end(&mut buf).unwrap(); + let (device, _) = Device::raw_decode(buf.as_slice()).unwrap(); + + Some(device) + } else { + None + } +} + +fn load_dev_vec(path: &str) -> Option> { + let mut dev_vec = Vec::new(); + match load_dev_by_path(path) { + Some(dev) => { + dev_vec.push(dev); + + Some(dev_vec) + }, + _ => None + } +} + +fn load_sn(sns: Vec<&str>) -> Option> { + let mut dev_vec = Vec::new(); + + if sns.len() == 0 { + match load_dev_by_path("sn-miner.desc") { + Some(dev) => { + dev_vec.push(dev); + Some(dev_vec) + }, + _ => None + } + } else { + for sn in sns { + let dev = load_dev_by_path(sn).unwrap(); + dev_vec.push(dev); + } + + Some(dev_vec) + } +} + +fn create_device(sns: Option>, endpoints: Vec) -> (Device, PrivateKey) { + let private_key = PrivateKey::generate_rsa(1024).unwrap(); + let public_key = private_key.public(); + + let sn_list = match sns.as_ref() { + Some(sns) => { + let mut sn_list = Vec::new(); + for sn in sns.iter() { + sn_list.push(sn.desc().device_id()); + } + sn_list + }, + None => vec![], + }; + + let device = Device::new( + None, + UniqueId::default(), + endpoints.clone(), + sn_list, + vec![], + public_key, + Area::default(), + DeviceCategory::PC + ).build(); + + (device, private_key) +} + +fn loger_init(log_level: &str, name: &str) { + if log_level != "none" { + cyfs_debug::CyfsLoggerBuilder::new_app(name) + .level(log_level) + .console(log_level) + .build() + .unwrap() + .start(); + + cyfs_debug::PanicBuilder::new(name, name) + .exit_on_panic(true) + .build() + .start(); + } +} + +pub fn command_line() -> clap::App<'static, 'static> { + App::new("bdt-tool") + .about("bdt tool") + .arg(Arg::with_name("ep").long("ep").multiple(true).takes_value(true).help("local endpoint")) + .arg(Arg::with_name("udp_sn_only").long("udp_sn_only").takes_value(false).default_value("0").help("udp sn only")) + .arg(Arg::with_name("log_level").long("log_level").default_value("none").help("log level: none/info/debug/warn/error")) + .arg(Arg::with_name("device_cache").long("device_cache").default_value("").help("device cache")) + .arg(Arg::with_name("sn").long("sn").multiple(true).default_value("").help("sn desc file")) + .arg(Arg::with_name("cmd").long("cmd").takes_value(false).help("sn desc file")) + .subcommand(SubCommand::with_name("ping") + .arg(Arg::with_name("remote").required(true)) + .arg(Arg::with_name("count").required(true)) + .arg(Arg::with_name("timeout").required(true)) + ) + .subcommand(SubCommand::with_name("nc") + .arg(Arg::with_name("remote").required(true)) + .arg(Arg::with_name("port").required(true)) + ) +} + +async fn remote_device( + stack: &Stack, + str: &str) -> BuckyResult { + if let Ok(device_id) = DeviceId::from_str(str) { + if let Some(device) = stack.device_cache().get(&device_id).await { + Ok(device) + } else { + Err(BuckyError::new(BuckyErrorCode::NotFound, "not found")) + } + } else { + let path = Path::new(str); + if !path.exists() { + Err(BuckyError::new(BuckyErrorCode::NotFound, "not found")) + } else { + let mut buf = vec![]; + let (device, _) = Device::decode_from_file(&path, &mut buf)?; + let device_id = device.desc().device_id(); + if stack.device_cache().get(&device_id).await.is_none() { + stack.device_cache().add(&device_id, &device); + } else { + } + Ok(device) + } + } +} + +#[async_std::main] +async fn main() { + // + let cmd_line = std::env::args().collect::>().join(" "); + let matches = command_line().get_matches(); + + let log_level = matches.value_of("log_level").unwrap(); + let udp_sn_only = u16::from_str(matches.value_of("udp_sn_only").unwrap()).unwrap(); + + let mut endpoints = vec![]; + for ep in matches.values_of("ep").unwrap() { + if let Ok(ep) = Endpoint::from_str(ep) { + endpoints.push(ep); + } else { + println!("invalid endpoint {}", ep); + return; + } + } + + let mut sns = vec![]; + for sn in matches.values_of("sn").unwrap() { + if sn.len() != 0 { + sns.push(sn); + } + } + let sns = load_sn(sns); + + //load device + let desc_path = Path::new("deamon.desc"); + let sec_path = Path::new("deamon.sec"); + if !sec_path.exists() { + println!("deamon.desc not exists, generate new one"); + + let (device, private_key) = create_device(sns.clone(), endpoints.clone()); + + if let Err(err) = device.encode_to_file(&desc_path, false) { + println!("generate deamon.desc failed for {}", err); + return; + } + if let Err(err) = private_key.encode_to_file(&sec_path, false) { + println!("generate deamon.sec failed for {}", err); + return; + } + } + if !desc_path.exists() { + println!("{:?} not exists", desc_path); + return; + } + let device = { + let mut buf = vec![]; + Device::decode_from_file(&desc_path, &mut buf).map(|(d, _)| d) + }; + if let Err(err) = device { + println!("load {:?} failed for {}", desc_path, err); + return; + } + let mut device = device.unwrap(); + info!("device={:?}", device); + + let private_key = { + let mut buf = vec![]; + PrivateKey::decode_from_file(&sec_path, &mut buf).map(|(k, _)| k) + }; + + if let Err(err) = private_key { + println!("load {:?} failed for {}", sec_path, err); + return; + } + let private_key = private_key.unwrap(); + + let deamon_id = device.desc().device_id(); + let deamon_name = format!("bdt-tool-{}", deamon_id); + loger_init(log_level, deamon_name.as_str()); + + let device_endpoints = device.mut_connect_info().mut_endpoints(); + device_endpoints.clear(); + for ep in endpoints { + device_endpoints.push(ep); + } + + //init stack + let mut params = StackOpenParams::new(deamon_name.as_str()); + let sns2 = sns.clone(); + params.known_sn = sns; + if udp_sn_only != 0 { + params.config.interface.udp.sn_only = true; + } else { + params.config.interface.udp.sn_only = false; + } + + let stack = Stack::open( + device, + private_key, + params).await; + + if let Err(err) = stack { + println!("open stack failed for {}", err); + return ; + } + + let stack = stack.unwrap(); + + if sns2.is_some() { + stack.reset_sn_list(sns2.unwrap()); + } + + match future::timeout( + Duration::from_secs(5), + stack.sn_client().ping().wait_online(), + ).await { + Ok(res) => { + match res { + Ok(res) => { + match res { + SnStatus::Online => { + }, + _ => { + println!("sn offline!"); + } + } + }, + Err(e) => { + println!("connect sn err={}", e); + } + } + }, + Err(e) => { + println!("wait_online err={}", e); + } + } + + if let Some(device_cache) = matches.value_of("device_cache") { + if device_cache.len() > 0 { + let dev = load_dev_by_path(device_cache).unwrap(); + let device_id = dev.desc().device_id(); + stack.device_cache().add(&device_id, &dev); + } + } + + // + let params = command_line().get_matches_from_safe(cmd_line.split(" ")) + .map_err(|err| err.message).unwrap(); + let subcommand = params.subcommand_name().ok_or_else(|| "no subcommand\r\n".to_string()).unwrap(); + match subcommand { + "ping" => { + let subcommand = params.subcommand_matches("ping").unwrap(); + let remote = remote_device(&stack, subcommand.value_of("remote").unwrap()).await + .map_err(|err| format!("load remote desc {} failed for {}\r\n", subcommand.value_of("remote").unwrap(), err)).unwrap(); + let count = u32::from_str(subcommand.value_of("count").unwrap()).unwrap(); + let timeout = u64::from_str(subcommand.value_of("timeout").unwrap()).unwrap(); + + let pinger = cyfs_bdt::debug::Pinger::open(stack.clone().to_weak()).unwrap(); + for _ in 0..count { + match pinger.ping(remote.clone(), Duration::from_secs(timeout), "debug".as_ref()).await { + Ok(rtt) => { + match rtt { + Some(rtt) => { + println!("ping success, rtt is {:.2} ms", rtt as f64 / 1000.0); + }, + None => { + println!("connected, but ping's seq mismatch"); + } + } + }, + Err(e) => { + println!("ping err={}", e); + } + } + } + + }, + "nc" => { + let subcommand = params.subcommand_matches("nc").unwrap(); + let remote = remote_device(&stack, subcommand.value_of("remote").unwrap()).await + .map_err(|err| format!("load remote desc {} failed for {}\r\n", subcommand.value_of("remote").unwrap(), err)).unwrap(); + let port = u16::from_str(subcommand.value_of("port").unwrap()).unwrap(); + let question = b"question?"; + match stack.stream_manager().connect( + port, + question.to_vec(), + BuildTunnelParams { + remote_const: remote.desc().clone(), + remote_sn: None, + remote_desc: Some(remote.clone()) + }).await { + Ok(conn) => { + println!("connect vport={} success!", port); + let _ = conn.shutdown(Shutdown::Both); + }, + Err(err) => { + println!("connect vport={} fail, err={}", port, err); + } + } + }, + _ => { + println!("unspport cmd {}", subcommand); + } + } +}