feat: move from sqlite to postgres

documentation/benchmark/main.py

@@ -28,7 +28,7 @@ def get_ip_address(ifname):
 CHECKER_ADDR = "http://127.0.0.1:16969"
 SERVICE_ADDR = get_ip_address("wlp3s0")
 
-VARIANTS = [0]
+VARIANTS = [0, 1]
 TICKS = 3
 MULTIPLIER = 1
 EXPLOITS_AMOUNT = 20

service/backend/database/init.go

@@ -2,32 +2,34 @@ package database
 
 import (
 	"log"
+	"time"
 
 	"replme/model"
+	"replme/util"
 
-	"gorm.io/driver/sqlite"
+	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 )
 
 var DB *gorm.DB
 
-func Connect(dbPath string) {
+func Connect(pgUrl string) {
 	var err error
-	DB, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{
-		SkipDefaultTransaction: true,
-		PrepareStmt:            true,
-	})
+	for i := 0; i < 10; i++ {
+		util.SLogger.Infof("Connecting to DB, try %d", i)
+		DB, err = gorm.Open(postgres.Open(pgUrl), &gorm.Config{
+			SkipDefaultTransaction: true,
+			PrepareStmt:            true,
+		})
+		if err == nil {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
 
 	if err != nil {
 		log.Fatal("Failed to connect to DB:", err)
 	}
-
-	DB.Raw("PRAGMA synchronous = NORMAL;")
-	DB.Raw("PRAGMA journal_mode = WAL;")
-	DB.Raw("PRAGMA temp_store = MEMORY;")
-	DB.Raw("PRAGMA cache_size = 10000;")
-	DB.Raw("PRAGMA mmap_size = 268435456;")
-	DB.Raw("PRAGMA optimize;")
 }
 
 func Migrate() {

service/backend/go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/otiai10/copy v1.14.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.22.0
-	gorm.io/driver/sqlite v1.4.4
+	gorm.io/driver/postgres v1.5.9
 	gorm.io/gorm v1.25.10
 )
 
@@ -41,14 +41,17 @@ require (
 	github.com/gorilla/context v1.1.2 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/sessions v1.2.2 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.5.5 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect

service/backend/go.sum

@@ -77,9 +77,16 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
+github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -100,9 +107,6 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
-github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
@@ -238,9 +242,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc=
-gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI=
-gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
+gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8=
+gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
 gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
 gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=

service/backend/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"os"
+	"strings"
 
 	"replme/server"
 	"replme/service"
@@ -12,15 +13,19 @@ import (
 func main() {
 	var imagePath string
 	var imageTag string
-	var dbPath string
+	var postgresUrl string
+	var postgresUser string
+	var postgresSecretPath string
 	var apiKeyPath string
 	var devenvsPath string
 	var devenvsTmpPath string
 	var containerLogsPath string
 
 	flag.StringVar(&imagePath, "i", "", "Image dir (required)")
 	flag.StringVar(&imagePath, "n", "", "Image tag (required), env: REPL_IMG_TAG")
-	flag.StringVar(&dbPath, "d", "", "Database file (required), env: REPL_SQLITE")
+	flag.StringVar(&postgresUrl, "p", "", "Postgres connection url (required), env: REPL_POSTGRES_URL")
+	flag.StringVar(&postgresUser, "u", "", "Postgres user (required), env: REPL_POSTGRES_USER")
+	flag.StringVar(&postgresSecretPath, "s", "", "Postgres secret file (required), env: REPL_POSTGRES_SECRET")
 	flag.StringVar(&apiKeyPath, "k", "", "Apikey file (required), env: REPL_API_KEY")
 	flag.StringVar(&devenvsPath, "f", "", "Devenv files dir (required), env: REPL_DEVENVS")
 	flag.StringVar(&devenvsTmpPath, "t", "", "Tmp devenv files dir (required), env: REPL_DEVENVS_TMP")
@@ -42,13 +47,31 @@ func main() {
 		imageTag = imageTagEnv
 	}
 
-	if dbPath == "" {
-		dbPathEnv := os.Getenv("REPL_SQLITE")
-		if dbPathEnv == "" {
+	if postgresUrl == "" {
+		postgresUrlEnv := os.Getenv("REPL_POSTGRES_URL")
+		if postgresUrlEnv == "" {
 			flag.Usage()
 			os.Exit(1)
 		}
-		dbPath = dbPathEnv
+		postgresUrl = postgresUrlEnv
+	}
+
+	if postgresUser == "" {
+		postgresUserEnv := os.Getenv("REPL_POSTGRES_USER")
+		if postgresUserEnv == "" {
+			flag.Usage()
+			os.Exit(1)
+		}
+		postgresUser = postgresUserEnv
+	}
+
+	if postgresSecretPath == "" {
+		postgresSecretPathEnv := os.Getenv("REPL_POSTGRES_SECRET")
+		if postgresSecretPathEnv == "" {
+			flag.Usage()
+			os.Exit(1)
+		}
+		postgresSecretPath = postgresSecretPathEnv
 	}
 
 	if apiKeyPath == "" {
@@ -78,8 +101,6 @@ func main() {
 		devenvsTmpPath = devenvsTmpPathEnv
 	}
 
-	// REPL_CONTAINER_LOGS
-
 	if containerLogsPath == "" {
 		containerLogsPathTmp := os.Getenv("REPL_CONTAINER_LOGS")
 		if containerLogsPathTmp == "" {
@@ -94,5 +115,9 @@ func main() {
 	docker := service.Docker(apiKey, imagePath, imageTag, containerLogsPath)
 	docker.BuildImage()
 
-	server.Init(&docker, dbPath, containerLogsPath, devenvsPath, devenvsTmpPath)
+	pgSecret := util.ReadPostgresSecret(postgresSecretPath)
+	pgUrl := strings.ReplaceAll(postgresUrl, "{user}", postgresUser)
+	pgUrl = strings.ReplaceAll(pgUrl, "{secret}", pgSecret)
+
+	server.Init(&docker, pgUrl, containerLogsPath, devenvsPath, devenvsTmpPath)
 }

service/backend/server/router.go

@@ -16,7 +16,7 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func NewRouter(docker *service.DockerService, dbPath string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) *gin.Engine {
+func NewRouter(docker *service.DockerService, pgUrl string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) *gin.Engine {
 
 	logLevel, exists := os.LookupEnv("REPL_LOG")
 	if !exists {
@@ -26,7 +26,7 @@ func NewRouter(docker *service.DockerService, dbPath string, containerLogsPath s
 	util.LoggerInit(logLevel)
 
 	util.SLogger.Info("Connecting to DB ..")
-	database.Connect(dbPath)
+	database.Connect(pgUrl)
 	util.SLogger.Info("Migrating DB ..")
 	database.Migrate()
 

service/backend/server/server.go

@@ -5,8 +5,8 @@ import (
 	"replme/util"
 )
 
-func Init(docker *service.DockerService, dbPath string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) {
-	engine := NewRouter(docker, dbPath, containerLogsPath, devenvFilesPath, devenvFilesTmpPath)
+func Init(docker *service.DockerService, pgUrl string, containerLogsPath string, devenvFilesPath string, devenvFilesTmpPath string) {
+	engine := NewRouter(docker, pgUrl, containerLogsPath, devenvFilesPath, devenvFilesTmpPath)
 	util.SLogger.Infof("Server is running on port 6969")
 	engine.Run(":6969")
 }

service/backend/util/rand.go

@@ -4,6 +4,8 @@ import (
 	"crypto/rand"
 	"math/big"
 	"os"
+	"strings"
+	"time"
 )
 
 const LETTERS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@@ -31,6 +33,19 @@ func RandomString(n int) (string, error) {
 	return string(ret), nil
 }
 
+func ReadPostgresSecret(postgresSecretPath string) string {
+	var bytes []byte
+	var err error
+	for i := 0; i < 20; i++ {
+		bytes, err = os.ReadFile(postgresSecretPath)
+		if err == nil {
+			return strings.TrimSpace(string(bytes))
+		}
+		time.Sleep(1 * time.Second)
+	}
+	panic(err)
+}
+
 func ApiKey(path string) string {
 	bytes, err := os.ReadFile(path)
 	if err != nil {

service/docker-compose.yml

@@ -12,26 +12,40 @@ services:
     container_name: replme-backend
     depends_on:
       - dind
+      - postgres
     build:
       context: .
       dockerfile: Dockerfile.backend
     volumes:
       - docker-cert:/cert
       - replme-data:/data
       - devenvs-tmp:/devenvs
+      - postgres-secret:/postgres
     environment:
       - DOCKER_CERT_PATH=/cert/client
       - DOCKER_TLS_VERIFY=true
       - GIN_MODE=release
       - REPL_LOG=debug
+      - REPL_POSTGRES_URL=postgres://{user}:{secret}@replme-postgres:5432
+      - REPL_POSTGRES_USER=postgres
+      - REPL_POSTGRES_SECRET=/postgres/secret
       - REPL_IMG_TAG=ptwhy
-      - REPL_SQLITE=/data/replme.db
       - REPL_API_KEY=/data/apikey
       - REPL_DEVENVS=/data/devenvs
       - REPL_DEVENVS_TMP=/devenvs
       - REPL_CONTAINER_LOGS=/data/logs
     restart: "unless-stopped"
 
+  postgres:
+    container_name: replme-postgres
+    build:
+      context: postgres
+      dockerfile: Dockerfile
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+      - postgres-secret:/root/data
+    restart: "unless-stopped"
+
   dind:
     image: docker:dind
     container_name: replme-dind
@@ -67,6 +81,8 @@ services:
 
 volumes:
   replme-data:
+  postgres-data:
+  postgres-secret:
   devenvs-tmp:
   docker-cert:
   docker-data:

service/postgres/Dockerfile

@@ -0,0 +1,10 @@
+FROM postgres:latest
+
+RUN apt-get update; apt-get install -y --no-install-recommends pwgen
+
+COPY docker-entrypoint-wrapped.sh /usr/local/bin
+
+RUN chown root:root /usr/local/bin/docker-entrypoint-wrapped.sh
+RUN chmod 500 /usr/local/bin/docker-entrypoint-wrapped.sh
+
+ENTRYPOINT ["docker-entrypoint-wrapped.sh"]

service/postgres/docker-entrypoint-wrapped.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+if [[ -e /root/data/secret ]]; then
+    POSTGRES_PASSWORD="$(cat /root/data/secret)"
+else
+    POSTGRES_PASSWORD="$(pwgen --numerals --capitalize --remove-chars="'\\" -1 32)"
+    echo -n "$POSTGRES_PASSWORD" > /root/data/secret
+fi
+export POSTGRES_PASSWORD
+
+docker-entrypoint.sh postgres