From 28009d33e2ad5669e75caf9d58efc1ec4bd03c78 Mon Sep 17 00:00:00 2001 From: Bohan Cheng <47214785+cbh778899@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:51:25 +1000 Subject: [PATCH] add functions to bind database volume to server (#82) Signed-off-by: cbh778899 --- .dockerignore | 3 ++- .env | 1 + .gitignore | 3 ++- Makefile | 2 ++ docker-compose-adv.yaml | 2 ++ docker-compose-dev.yaml | 1 + docker-compose.yaml | 2 ++ setup/config.h | 2 ++ setup/default_config.h | 2 ++ setup/setup.c | 23 ++++++++++++++++------- setup/setup_types.h | 29 +++++++++++++++++++++++------ 11 files changed, 55 insertions(+), 15 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4588173..92c438b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,4 +14,5 @@ volumes docker-compose* Makefile setup -generate_production_env.html \ No newline at end of file +generate_production_env.html +landedb \ No newline at end of file diff --git a/.env b/.env index 266e34b..216b915 100644 --- a/.env +++ b/.env @@ -2,6 +2,7 @@ APP_PORT=8000 APP_EXPOSE_PORT=8000 ENG_ACCESS_PORT=8080 MODEL_SAVE_PATH=volumes/models +DATABASE_BIND_PATH=./lancedb INFERENCE_ENG=llamacpp INFERENCE_ENG_PORT=8080 INFERENCE_ENG_VERSION=server--b1-27d4b7c diff --git a/.gitignore b/.gitignore index a7f452c..38c36bb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules .git volumes __pycache__ -setup/setup \ No newline at end of file +setup/setup +lancedb \ No newline at end of file diff --git a/Makefile b/Makefile index 223f665..c71a7f3 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ ENV_FILE:=.env ENG_ACCESS_PORT:=8080 MODEL_SAVE_PATH:=volumes/models +DATABASE_BIND_PATH:=./lancedb INFERENCE_ENG:=llamacpp INFERENCE_ENG_PORT:=8080 @@ -40,6 +41,7 @@ env: @echo "APP_EXPOSE_PORT=$(APP_EXPOSE_PORT)">> $(ENV_FILE) @echo "ENG_ACCESS_PORT=$(ENG_ACCESS_PORT)">> $(ENV_FILE) @echo "MODEL_SAVE_PATH=$(MODEL_SAVE_PATH)">> $(ENV_FILE) + @echo "DATABASE_BIND_PATH=$(DATABASE_BIND_PATH)">> $(ENV_FILE) @echo "INFERENCE_ENG=$(INFERENCE_ENG)">> $(ENV_FILE) @echo "INFERENCE_ENG_PORT=$(INFERENCE_ENG_PORT)">> $(ENV_FILE) @echo "INFERENCE_ENG_VERSION=$(INFERENCE_ENG_VERSION)">> $(ENV_FILE) diff --git a/docker-compose-adv.yaml b/docker-compose-adv.yaml index 8af175f..301911c 100644 --- a/docker-compose-adv.yaml +++ b/docker-compose-adv.yaml @@ -37,6 +37,8 @@ services: build: dockerfile: setup/Dockerfile context: . + volumes: + - ${DATABASE_BIND_PATH}:/tmp/lancedb expose: - ${APP_EXPOSE_PORT} ports: diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index f848fdd..7e6d5d6 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -39,6 +39,7 @@ services: context: . volumes: - .:/app + - ${DATABASE_BIND_PATH}:/tmp/lancedb expose: - ${APP_PORT} ports: diff --git a/docker-compose.yaml b/docker-compose.yaml index 8b44069..ffd885d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -37,6 +37,8 @@ services: build: dockerfile: Dockerfile context: . + volumes: + - ${DATABASE_BIND_PATH}:/tmp/lancedb expose: - ${APP_PORT} ports: diff --git a/setup/config.h b/setup/config.h index 6a48c69..b701e79 100644 --- a/setup/config.h +++ b/setup/config.h @@ -44,4 +44,6 @@ #define DEV_MODE_ENABLED 0 +#define DATABASE_BIND_PATH "./lancedb" + #endif \ No newline at end of file diff --git a/setup/default_config.h b/setup/default_config.h index e9fc66b..5983c86 100644 --- a/setup/default_config.h +++ b/setup/default_config.h @@ -44,4 +44,6 @@ #define DEFAULT_DEV_MODE_ENABLED 0 +#define DEFAULT_DATABASE_BIND_PATH "./lancedb" + #endif \ No newline at end of file diff --git a/setup/setup.c b/setup/setup.c index 8a1a3be..0f9de59 100644 --- a/setup/setup.c +++ b/setup/setup.c @@ -41,6 +41,8 @@ int api_version_enabled = API_VERSION_ENABLED; int dev_mode_enabled = DEV_MODE_ENABLED; +char* database_bind_path = DATABASE_BIND_PATH; + // ===============================FUNCTION DECLARATION=============================== void logMenu(char*); @@ -128,14 +130,21 @@ void handleMenuSelection() { defaultDatasetSettings(); break; case 'h': case 'H': - staticAPIKeySettings(); + setStringFields( + DATABSE_PATH_MENU, + "Set Database Path Success", + "Database path: ", + &database_bind_path + ); break; case 'i': case 'I': - apiSettings(); + staticAPIKeySettings(); break; case 'j': case 'J': - setYesNoFields(DEV_MODE_MENU, &dev_mode_enabled); break; + apiSettings(); case 'k': case 'K': + setYesNoFields(DEV_MODE_MENU, &dev_mode_enabled); break; + case 'l': case 'L': buildSettings(); break; case 'q': case 'Q': case ESC: puts("Exit."); @@ -591,7 +600,8 @@ void saveSettings(int show_message) { api_embedding_calc_enabled, api_embedding_ds_enabled, api_version_enabled, - dev_mode_enabled + dev_mode_enabled, + database_bind_path ); fclose(f); @@ -599,6 +609,7 @@ void saveSettings(int show_message) { f = fopen(".env", "w"); fprintf(f, ENV_FILE, app_expose_port, + database_bind_path, inference_cpu_cores, inference_thread_counts, embedding_cpu_cores, @@ -671,8 +682,6 @@ void saveSettings(int show_message) { https_cert_path_host, https_ca_name, https_cert_path_container, https_ca_name ); } - // check if should bind any volume to host machine - int volume_bind_check = dev_mode_enabled || secret_bind_check; int static_api_key_availibility = static_api_key_enabled && @@ -683,7 +692,6 @@ void saveSettings(int show_message) { fprintf(f, DOCKER_COMPOSE_FILE, static_api_key_availibility ? static_api_key_str : "", - volume_bind_check ? COMPOSE_FILE_VOLUME_SECTION : "", dev_mode_enabled ? COMPOSE_FILE_DEV_MODE : "", secret_bind_check ? docker_compose_ssl_str : "" ); @@ -755,5 +763,6 @@ void saveDefaultSettings() { api_embedding_ds_enabled = DEFAULT_API_EMBEDDING_DS_ENABLED; api_version_enabled = DEFAULT_API_VERSION_ENABLED; dev_mode_enabled = DEFAULT_DEV_MODE_ENABLED; + database_bind_path = DEFAULT_DATABASE_BIND_PATH; saveSettings(1); } \ No newline at end of file diff --git a/setup/setup_types.h b/setup/setup_types.h index 8befe64..c55d72e 100644 --- a/setup/setup_types.h +++ b/setup/setup_types.h @@ -34,10 +34,11 @@ "e. Set Plugin Availability\n"\ "f. Set System Instruction\n"\ "g. Set Default Dataset Features\n"\ - "h. Set Static API-Key Features\n"\ - "i. Set APIs Availability\n"\ - "j. Toggle Developer Mode\n"\ - "k. Save & Build\n"\ + "h. Set Database Path on Server\n"\ + "i. Set Static API-Key Features\n"\ + "j. Set APIs Availability\n"\ + "k. Toggle Developer Mode\n"\ + "l. Save & Build\n"\ "q. Quit\n"\ "\n"\ "Please enter your selection: " @@ -437,6 +438,18 @@ "Enter 'q' to go back\n\n"\ "Do you want to enable developer mode? (Y/N): " +// ===============================DEV MODE=============================== +#define DATABSE_PATH_MENU \ + "================================\n"\ + "\n"\ + " Database Path on Server\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Please set your database path on server, so you won't lost your data when you restart your server.\n"\ + "Enter 'q' to go back\n\n"\ + "Path of database: " + // ===============================CONFIGS=============================== #define SAVE_CONFIG_SUCCESS "Successfully Saved Config" @@ -501,6 +514,8 @@ "\n"\ "#define DEV_MODE_ENABLED %d\n"\ "\n"\ +"#define DATABASE_BIND_PATH \"%s\"\n"\ +"\n"\ "#endif" #define DOCKER_COMPOSE_FILE \ @@ -544,7 +559,9 @@ " dockerfile: setup/Dockerfile\n"\ " context: .\n"\ "%s"\ -"%s%s%s"\ +" volumes:\n"\ +" - ${DATABASE_BIND_PATH}:/tmp/lancedb\n"\ +"%s%s"\ " expose:\n"\ " - ${APP_EXPOSE_PORT}\n"\ " ports:\n"\ @@ -553,7 +570,6 @@ " - llamacpp\n"\ " - embedding_eng\n"\ -#define COMPOSE_FILE_VOLUME_SECTION " volumes:\n" #define COMPOSE_FILE_DEV_MODE " - .:/app\n" #define COMPOSE_FILE_STATIC_API_KEY \ " environment:\n"\ @@ -569,6 +585,7 @@ "APP_EXPOSE_PORT=%s\n"\ "ENG_ACCESS_PORT=8080\n"\ "MODEL_SAVE_PATH=volumes/models\n"\ +"DATABASE_BIND_PATH=%s\n"\ "INFERENCE_ENG=llamacpp\n"\ "INFERENCE_ENG_PORT=8080\n"\ "INFERENCE_ENG_VERSION=server--b1-27d4b7c\n"\