From add91dbba225e87c2c31a8276d84cbcc30ccd710 Mon Sep 17 00:00:00 2001 From: Jacob Bachmann Date: Sun, 14 Jul 2024 16:54:23 +0200 Subject: [PATCH] feat: move from sqlite to postgres --- documentation/benchmark/main.py | 2 +- service/backend/database/init.go | 28 ++++++------ service/backend/go.mod | 7 ++- service/backend/go.sum | 17 +++++--- service/backend/main.go | 43 +++++++++++++++---- service/backend/server/router.go | 4 +- service/backend/server/server.go | 4 +- service/backend/util/rand.go | 15 +++++++ service/docker-compose.yml | 18 +++++++- service/postgres/Dockerfile | 10 +++++ service/postgres/docker-entrypoint-wrapped.sh | 11 +++++ 11 files changed, 122 insertions(+), 37 deletions(-) create mode 100644 service/postgres/Dockerfile create mode 100644 service/postgres/docker-entrypoint-wrapped.sh diff --git a/documentation/benchmark/main.py b/documentation/benchmark/main.py index b79cb57..9c6bd2e 100644 --- a/documentation/benchmark/main.py +++ b/documentation/benchmark/main.py @@ -28,7 +28,7 @@ def get_ip_address(ifname): CHECKER_ADDR = "http://127.0.0.1:16969" SERVICE_ADDR = get_ip_address("wlp3s0") -VARIANTS = [0] +VARIANTS = [0, 1] TICKS = 3 MULTIPLIER = 1 EXPLOITS_AMOUNT = 20 diff --git a/service/backend/database/init.go b/service/backend/database/init.go index 4411a75..2c7bb16 100644 --- a/service/backend/database/init.go +++ b/service/backend/database/init.go @@ -2,32 +2,34 @@ package database import ( "log" + "time" "replme/model" + "replme/util" - "gorm.io/driver/sqlite" + "gorm.io/driver/postgres" "gorm.io/gorm" ) var DB *gorm.DB -func Connect(dbPath string) { +func Connect(pgUrl string) { var err error - DB, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{ - SkipDefaultTransaction: true, - PrepareStmt: true, - }) + for i := 0; i < 10; i++ { + util.SLogger.Infof("Connecting to DB, try %d", i) + DB, err = gorm.Open(postgres.Open(pgUrl), &gorm.Config{ + SkipDefaultTransaction: true, + PrepareStmt: true, + }) + if err == nil { + break + } + time.Sleep(1 * time.Second) + } if err != nil { log.Fatal("Failed to connect to DB:", err) } - - DB.Raw("PRAGMA synchronous = NORMAL;") - DB.Raw("PRAGMA journal_mode = WAL;") - DB.Raw("PRAGMA temp_store = MEMORY;") - DB.Raw("PRAGMA cache_size = 10000;") - DB.Raw("PRAGMA mmap_size = 268435456;") - DB.Raw("PRAGMA optimize;") } func Migrate() { diff --git a/service/backend/go.mod b/service/backend/go.mod index 099472e..ccde005 100644 --- a/service/backend/go.mod +++ b/service/backend/go.mod @@ -13,7 +13,7 @@ require ( github.com/otiai10/copy v1.14.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.22.0 - gorm.io/driver/sqlite v1.4.4 + gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.10 ) @@ -41,6 +41,10 @@ require ( github.com/gorilla/context v1.1.2 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.2.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,7 +52,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect diff --git a/service/backend/go.sum b/service/backend/go.sum index a6a6963..27e4bf1 100644 --- a/service/backend/go.sum +++ b/service/backend/go.sum @@ -77,9 +77,16 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -100,9 +107,6 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= @@ -238,9 +242,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc= -gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= -gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= +gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= diff --git a/service/backend/main.go b/service/backend/main.go index c304d2a..13cd2a0 100644 --- a/service/backend/main.go +++ b/service/backend/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "os" + "strings" "replme/server" "replme/service" @@ -12,7 +13,9 @@ import ( func main() { var imagePath string var imageTag string - var dbPath string + var postgresUrl string + var postgresUser string + var postgresSecretPath string var apiKeyPath string var devenvsPath string var devenvsTmpPath string @@ -20,7 +23,9 @@ func main() { flag.StringVar(&imagePath, "i", "", "Image dir (required)") flag.StringVar(&imagePath, "n", "", "Image tag (required), env: REPL_IMG_TAG") - flag.StringVar(&dbPath, "d", "", "Database file (required), env: REPL_SQLITE") + flag.StringVar(&postgresUrl, "p", "", "Postgres connection url (required), env: REPL_POSTGRES_URL") + flag.StringVar(&postgresUser, "u", "", "Postgres user (required), env: REPL_POSTGRES_USER") + flag.StringVar(&postgresSecretPath, "s", "", "Postgres secret file (required), env: REPL_POSTGRES_SECRET") flag.StringVar(&apiKeyPath, "k", "", "Apikey file (required), env: REPL_API_KEY") flag.StringVar(&devenvsPath, "f", "", "Devenv files dir (required), env: REPL_DEVENVS") flag.StringVar(&devenvsTmpPath, "t", "", "Tmp devenv files dir (required), env: REPL_DEVENVS_TMP") @@ -42,13 +47,31 @@ func main() { imageTag = imageTagEnv } - if dbPath == "" { - dbPathEnv := os.Getenv("REPL_SQLITE") - if dbPathEnv == "" { + if postgresUrl == "" { + postgresUrlEnv := os.Getenv("REPL_POSTGRES_URL") + if postgresUrlEnv == "" { flag.Usage() os.Exit(1) } - dbPath = dbPathEnv + postgresUrl = postgresUrlEnv + } + + if postgresUser == "" { + postgresUserEnv := os.Getenv("REPL_POSTGRES_USER") + if postgresUserEnv == "" { + flag.Usage() + os.Exit(1) + } + postgresUser = postgresUserEnv + } + + if postgresSecretPath == "" { + postgresSecretPathEnv := os.Getenv("REPL_POSTGRES_SECRET") + if postgresSecretPathEnv == "" { + flag.Usage() + os.Exit(1) + } + postgresSecretPath = postgresSecretPathEnv } if apiKeyPath == "" { @@ -78,8 +101,6 @@ func main() { devenvsTmpPath = devenvsTmpPathEnv } - // REPL_CONTAINER_LOGS - if containerLogsPath == "" { containerLogsPathTmp := os.Getenv("REPL_CONTAINER_LOGS") if containerLogsPathTmp == "" { @@ -94,5 +115,9 @@ func main() { docker := service.Docker(apiKey, imagePath, imageTag, containerLogsPath) docker.BuildImage() - server.Init(&docker, dbPath, containerLogsPath, devenvsPath, devenvsTmpPath) + pgSecret := util.ReadPostgresSecret(postgresSecretPath) + pgUrl := strings.ReplaceAll(postgresUrl, "{user}", postgresUser) + pgUrl = strings.ReplaceAll(pgUrl, "{secret}", pgSecret) + + server.Init(&docker, pgUrl, containerLogsPath, devenvsPath, devenvsTmpPath) } diff --git a/service/backend/server/router.go b/service/backend/server/router.go index d5de5db..7592ec6 100644 --- a/service/backend/server/router.go +++ b/service/backend/server/router.go @@ -16,7 +16,7 @@ import ( "github.com/gin-gonic/gin" ) -func NewRouter(docker *service.DockerService, dbPath string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) *gin.Engine { +func NewRouter(docker *service.DockerService, pgUrl string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) *gin.Engine { logLevel, exists := os.LookupEnv("REPL_LOG") if !exists { @@ -26,7 +26,7 @@ func NewRouter(docker *service.DockerService, dbPath string, containerLogsPath s util.LoggerInit(logLevel) util.SLogger.Info("Connecting to DB ..") - database.Connect(dbPath) + database.Connect(pgUrl) util.SLogger.Info("Migrating DB ..") database.Migrate() diff --git a/service/backend/server/server.go b/service/backend/server/server.go index 7445443..7857994 100644 --- a/service/backend/server/server.go +++ b/service/backend/server/server.go @@ -5,8 +5,8 @@ import ( "replme/util" ) -func Init(docker *service.DockerService, dbPath string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) { - engine := NewRouter(docker, dbPath, containerLogsPath, devenvFilesPath, devenvFilesTmpPath) +func Init(docker *service.DockerService, pgUrl string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) { + engine := NewRouter(docker, pgUrl, containerLogsPath, devenvFilesPath, devenvFilesTmpPath) util.SLogger.Infof("Server is running on port 6969") engine.Run(":6969") } diff --git a/service/backend/util/rand.go b/service/backend/util/rand.go index 9596b2d..6c2d9ad 100644 --- a/service/backend/util/rand.go +++ b/service/backend/util/rand.go @@ -4,6 +4,8 @@ import ( "crypto/rand" "math/big" "os" + "strings" + "time" ) const LETTERS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" @@ -31,6 +33,19 @@ func RandomString(n int) (string, error) { return string(ret), nil } +func ReadPostgresSecret(postgresSecretPath string) string { + var bytes []byte + var err error + for i := 0; i < 20; i++ { + bytes, err = os.ReadFile(postgresSecretPath) + if err == nil { + return strings.TrimSpace(string(bytes)) + } + time.Sleep(1 * time.Second) + } + panic(err) +} + func ApiKey(path string) string { bytes, err := os.ReadFile(path) if err != nil { diff --git a/service/docker-compose.yml b/service/docker-compose.yml index 61a08d8..c3f5dcc 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -12,6 +12,7 @@ services: container_name: replme-backend depends_on: - dind + - postgres build: context: . dockerfile: Dockerfile.backend @@ -19,19 +20,32 @@ services: - docker-cert:/cert - replme-data:/data - devenvs-tmp:/devenvs + - postgres-secret:/postgres environment: - DOCKER_CERT_PATH=/cert/client - DOCKER_TLS_VERIFY=true - GIN_MODE=release - REPL_LOG=debug + - REPL_POSTGRES_URL=postgres://{user}:{secret}@replme-postgres:5432 + - REPL_POSTGRES_USER=postgres + - REPL_POSTGRES_SECRET=/postgres/secret - REPL_IMG_TAG=ptwhy - - REPL_SQLITE=/data/replme.db - REPL_API_KEY=/data/apikey - REPL_DEVENVS=/data/devenvs - REPL_DEVENVS_TMP=/devenvs - REPL_CONTAINER_LOGS=/data/logs restart: "unless-stopped" + postgres: + container_name: replme-postgres + build: + context: postgres + dockerfile: Dockerfile + volumes: + - postgres-data:/var/lib/postgresql/data + - postgres-secret:/root/data + restart: "unless-stopped" + dind: image: docker:dind container_name: replme-dind @@ -67,6 +81,8 @@ services: volumes: replme-data: + postgres-data: + postgres-secret: devenvs-tmp: docker-cert: docker-data: diff --git a/service/postgres/Dockerfile b/service/postgres/Dockerfile new file mode 100644 index 0000000..a481d82 --- /dev/null +++ b/service/postgres/Dockerfile @@ -0,0 +1,10 @@ +FROM postgres:latest + +RUN apt-get update; apt-get install -y --no-install-recommends pwgen + +COPY docker-entrypoint-wrapped.sh /usr/local/bin + +RUN chown root:root /usr/local/bin/docker-entrypoint-wrapped.sh +RUN chmod 500 /usr/local/bin/docker-entrypoint-wrapped.sh + +ENTRYPOINT ["docker-entrypoint-wrapped.sh"] diff --git a/service/postgres/docker-entrypoint-wrapped.sh b/service/postgres/docker-entrypoint-wrapped.sh new file mode 100644 index 0000000..9ba55a1 --- /dev/null +++ b/service/postgres/docker-entrypoint-wrapped.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if [[ -e /root/data/secret ]]; then + POSTGRES_PASSWORD="$(cat /root/data/secret)" +else + POSTGRES_PASSWORD="$(pwgen --numerals --capitalize --remove-chars="'\\" -1 32)" + echo -n "$POSTGRES_PASSWORD" > /root/data/secret +fi +export POSTGRES_PASSWORD + +docker-entrypoint.sh postgres