Merge pull request #213 from benjamin-747/main

added use config to init repo directory

.env

@@ -7,24 +7,27 @@ MEGA_DB_MYSQL_URL = "mysql://${MYSQL_USERNAME}:${MYSQL_SECRET}@${MYSQL_HOST}/meg
 MEGA_DB_MAX_CONNECTIONS = 32
 MEGA_DB_MIN_CONNECTIONS = 16
 
-MEGA_DB_SQLX_LOGGING = false # Whether to disabling SQLx Log
+MEGA_DB_SQLX_LOGGING = false # Whether to enabling SQLx Log
 
-# If the object file size exceeds the threshold value, it will be stored in the specified location instead of the database.
-MEGA_BIG_OBJ_THRESHOLD_SIZE = 1024 # Unit KB
-MEGA_BIG_OBJ_STORAGR_PATH = "/tmp/.mega/objects"
+## file storage configuration
+MEGA_OBJ_STORAGR_TYPE = "LOCAL" # LOCAL or REMOTE
+MEGA_OBJ_LOCAL_PATH = "/tmp/.mega/objects" # This configuration is used to set the local location of the objetcs storage
 
+MEGA_BIG_OBJ_THRESHOLD_SIZE = 1024 # Unit KB. If the object file size exceeds the threshold value, it will be handled by file storage instead of the database.
 
-MGEA_LFS_FILE_LOCAL_PATH = "/tmp/.mega/objects" # This configuration is used to set the local location of the lfs store
+## Init directory configuration
+MEGA_INIT_DIRS = "/projects,/docs,/third_parts" # init these repo directories in mega init command
+MEGA_IMPORT_DIRS = "/third_parts" # Only import directory support multi-branch commit and tag, repo under regular directory only support main branch only
 
+## Objects decode configuration
 GIT_INTERNAL_DECODE_CACHE_SIZE = 1000 # Maximum number of git objects in LRU cache
 GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE = 10000 # The maximum number of git object in a "INSERT" SQL database operation
 GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE = 10 # The maximum number of parallel insertion threads in the database operation queue
 GIT_INTERNAL_DECODE_CACHE_TYEP = "lru" #{lru,redis}
 REDIS_CONFIG = "redis://127.0.0.1:6379"
 
-
-## Bazel build config, you can use service like buildfarm to enable RBE(remote build execution)
-# you can refer to https://bazelbuild.github.io/bazel-buildfarm/docs/quick_start/ for more details about remote executor
+## Bazel build configuration
+## you can use service like buildfarm to enable RBE(remote build execution), refer to https://bazelbuild.github.io/bazel-buildfarm/docs/quick_start/ for more details about remote executor
 BAZEL_BUILD_ENABLE = false # leave true if you want to trigger bazel build in each push process
 BAZEL_BUILDP_PATH = "/tmp/.mega/bazel_build_projects" # Specify a temporary directory to build the project with bazel
 BAZEL_REMOTE_EXECUTOR = "grpc://localhost:8980" # If enable the remote executor, please fillin the remote executor address, or else leave empty if you want to build by localhost. 

common/src/errors.rs

@@ -35,6 +35,13 @@ impl MegaError {
             code: 1,
         }
     }
+
+    pub fn with_message(msg: &str) -> MegaError {
+        MegaError {
+            error: anyhow::anyhow!("Error Message: {}", msg).into(),
+            code: 0,
+        }
+    }
 }
 
 impl std::fmt::Display for MegaError {

git/src/lfs/http.rs

@@ -426,8 +426,8 @@ pub async fn lfs_upload_object(
         request_body.extend_from_slice(&bytes);
     }
 
-    let ok = content_store.put(&meta.oid, meta.size, request_body.freeze().as_ref());
-    if !ok {
+    let res = content_store.put(&meta.oid, meta.size, request_body.freeze().as_ref());
+    if !res.is_ok() {
         config.storage.lfs_delete_meta(&request_vars).await.unwrap();
         return Err((
             StatusCode::NOT_ACCEPTABLE,

src/commands/https.rs

@@ -23,7 +23,7 @@ pub(crate) async fn exec(_config: Config, args: &ArgMatches) -> MegaResult {
         .unwrap();
     if server_matchers.lfs_content_path.is_none() {
         server_matchers.lfs_content_path =
-            Some(PathBuf::from(env::var("MGEA_LFS_FILE_LOCAL_PATH").unwrap()))
+            Some(PathBuf::from(env::var("MEGA_OBJ_LOCAL_PATH").unwrap()))
     }
     println!("{server_matchers:#?}");
     https::http_server(&server_matchers).await.unwrap();

storage/src/driver/database/storage.rs

@@ -42,9 +42,11 @@ use sea_orm::QuerySelect;
 use sea_orm::Set;
 use sea_orm::TryIntoModel;
 
-use crate::driver::fs::local_storage::MetaObject;
 use crate::driver::fs::lfs_structs::Lock;
 use crate::driver::fs::lfs_structs::RequestVars;
+use crate::driver::fs::local_storage::LocalStorage;
+use crate::driver::fs::local_storage::MetaObject;
+use crate::driver::fs::FileStorage;
 use common::errors::GitLFSError;
 use common::errors::MegaError;
 
@@ -74,23 +76,20 @@ pub trait ObjectStorage: Send + Sync {
             .parse::<usize>()
             .unwrap();
 
-        let storage_path = env::var("MEGA_BIG_OBJ_STORAGR_PATH")
-            .expect("MEGA_BIG_OBJ_STORAGR_PATH not configured")
+        let storage_path = env::var("MEGA_OBJ_LOCAL_PATH")
+            .expect("MEGA_OBJ_LOCAL_PATH not configured")
             .parse::<PathBuf>()
             .unwrap();
+        let storage = LocalStorage::init(storage_path);
+
         let mut new_obj_data: Vec<git_obj::ActiveModel> = Vec::new();
         for model in obj_data.iter_mut() {
             let mut obj = model.clone().try_into_model().unwrap();
             if obj.data.len() / 1024 > threshold {
-                let git_id = &obj.git_id;
-                let mut full_path = storage_path.clone();
-                full_path.push(&git_id[0..2]);
-                full_path.push(&git_id[2..4]);
-                fs::create_dir_all(&full_path).unwrap();
-                full_path.push(git_id);
-                let mut obj_file = File::create(&full_path).unwrap();
-                obj_file.write_all(&obj.data).unwrap();
-                obj.link = full_path.to_str().map(|s| s.to_string());
+                let path = storage
+                    .put(&obj.git_id, obj.data.len() as i64, &obj.data)
+                    .unwrap();
+                obj.link = Some(path);
                 obj.data.clear();
             }
             new_obj_data.push(obj.into_active_model())
@@ -634,14 +633,26 @@ pub trait ObjectStorage: Send + Sync {
             let root = repo_directory::new(0, "root", "/");
             self.save_directory(root).await?
         };
-        let docs = repo_directory::new(pid, "docs", "/docs");
-        let third_parts = repo_directory::new(pid, "third_parts", "/third_parts");
-        let projects = repo_directory::new(pid, "projects", "/projects");
-        let model_vec = vec![docs, third_parts, projects];
+
+        let init_dirs = env::var("MEGA_INIT_DIRS").unwrap();
+        let mut model_vec = Vec::new();
+        for str in init_dirs.split(',') {
+            let path = PathBuf::from(str);
+            model_vec.push(repo_directory::new(
+                pid,
+                path.file_name().unwrap().to_str().unwrap(),
+                str,
+            ));
+        }
         repo_directory::Entity::insert_many(model_vec)
             .on_conflict(
                 OnConflict::column(repo_directory::Column::FullPath)
-                    .do_nothing()
+                    .update_columns([
+                        repo_directory::Column::Pid,
+                        repo_directory::Column::Name,
+                        repo_directory::Column::IsRepo,
+                        repo_directory::Column::UpdatedAt,
+                    ])
                     .to_owned(),
             )
             .exec(self.get_connection())

storage/src/driver/fs/local_storage.rs

@@ -1,3 +1,4 @@
+use common::errors::MegaError;
 use sha256::digest;
 use std::fs;
 use std::io::prelude::*;
@@ -30,22 +31,21 @@ impl FileStorage for LocalStorage {
         fs::File::open(path).expect("Open file failed!")
     }
 
-    fn put(&self, object_id: &str, size: i64, body_content: &[u8]) -> bool {
+    fn put(&self, object_id: &str, size: i64, body_content: &[u8]) -> Result<String, MegaError> {
         let path = path::Path::new(&self.base_path).join(Self::transform_path(object_id));
         let dir = path.parent().unwrap();
         fs::create_dir_all(dir).expect("Create directory failed!");
 
         let mut file = fs::File::create(&path).expect("Open file failed");
         let lenght_written = file.write(body_content).expect("Write file failed");
         if lenght_written as i64 != size {
-            return false;
+            return Err(MegaError::with_message("size not correct"));
         }
-
         let hash = digest(body_content);
         if hash != object_id {
-            return false;
+            return Err(MegaError::with_message("hash not matched"));
         }
-        true
+        Ok(path.to_str().unwrap().to_string())
     }
 
     fn exist(&self, object_id: &str) -> bool {
@@ -74,8 +74,8 @@ mod tests {
         let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap());
         source.push("tests/objects");
 
-        let local_storage = LocalStorage::init(source);
-        assert!(local_storage.put(&meta.oid, meta.size, content));
+        let local_storage = LocalStorage::init(source.clone());
+        assert!(local_storage.put(&meta.oid, meta.size, content).is_ok());
 
         assert!(local_storage.exist(&meta.oid));
     }

storage/src/driver/fs/mod.rs

@@ -1,4 +1,9 @@
-use std::{fs::File, path};
+use std::{
+    fs::File,
+    path::{self},
+};
+
+use common::errors::MegaError;
 
 pub mod lfs_structs;
 pub mod local_storage;
@@ -7,7 +12,7 @@ pub mod remote_storage;
 pub trait FileStorage {
     fn get(&self, object_id: &str) -> File;
 
-    fn put(&self, object_id: &str, size: i64, body_content: &[u8]) -> bool;
+    fn put(&self, object_id: &str, size: i64, body_content: &[u8]) -> Result<String, MegaError>;
 
     fn exist(&self, object_id: &str) -> bool;