Development (#45)

* Update README.md * Redis credentials (#44) * Update issue templates * OSS (#42) * removing sensitive information for opensourcing the project * add chart for deploying phoenix * remove hardcoded port * Update README.md * Create docker-publish.yml * Update README.md * Update README.md * Create LICENSE * Update README.md * Guidelines (#43) * Create CODE_OF_CONDUCT.md * Create CONTRIBUTING.md * Update README.md * Update README.md * Create USERS.md * Update README.md * Update README.md * bump redis client version, add DB_PASSWORD for redis password, fix worker queue * typo in error checking * add new ENV to chart Co-authored-by: Niels ten Boom <[email protected]> * remove default password from tests and cli Co-authored-by: Niels ten Boom <[email protected]>
rtlnl · Nov 2, 2020 · 18efad5 · 18efad5
1 parent 3afa1ab
commit 18efad5
Show file tree

Hide file tree

Showing 18 changed files with 107 additions and 124 deletions.
diff --git a/README.md b/README.md
@@ -34,4 +34,4 @@ In order to contribute to Phoenix, see the [CONTRIBUTING](CONTRIBUTING.md) file
 If your company or your product is using Phoenix, please let us know by adding yourself to the Phoenix [users](USERS.md) file.
 
 ## License
-Phoenix is licensed under MIT license as found in the [LICENSE](LICENSE.md) file.
+Phoenix is licensed under MIT license as found in the [LICENSE](LICENSE.md) file.
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -25,6 +25,7 @@ internal:
   data:
     # all the env variables can be found here: https://github.com/rtlnl/phoenix/blob/master/cmd/internal.go#L64
     DB_HOST: "redis-master.phoenix:6379"
+    DB_PASSWORD: ""
     S3_REGION: "us-west-1"
     S3_ENDPOINT: "s3.eu-west-1.amazonaws.com"
     S3_DISABLE_SSL: "false"
@@ -43,6 +44,7 @@ public:
 
   data:
     DB_HOST: "redis-master.phoenix:6379"
+    DB_PASSWORD: ""
     # Other ENV you can set in the configmap
     # You can find all the possible variables here: https://github.com/rtlnl/phoenix/blob/master/cmd/public.go#L102
     # REC_LOGS_TYPE: "kafka"
@@ -65,7 +67,8 @@ worker:
 
   data:
     # all the env variables can be found here: https://github.com/rtlnl/phoenix/blob/master/cmd/worker.go#L79
-    WORKER_BROKER_URL: "edis-master.phoenix:6379"
+    WORKER_BROKER_URL: "redis-master.phoenix:6379"
+    WORKER_PASSWORD: ""
 
   resources: {}
   tolerations: {}

diff --git a/cmd/internal.go b/cmd/internal.go
@@ -26,6 +26,7 @@ var internalCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		addr := viper.GetString(addressInternalFlag)
 		dbHost := viper.GetString(dbHostInternalFlag)
+		dbPassword := viper.GetString(dbPasswordInternalFlag)
 		s3Region := viper.GetString(s3RegionFlag)
 		s3Endpoint := viper.GetString(s3EndpointFlag)
 		s3DisableSSL := viper.GetBool(s3DisableSSLFlag)
@@ -38,7 +39,7 @@ var internalCmd = &cobra.Command{
 		}
 
 		// instantiate Redis client
-		redisClient, err := db.NewRedisClient(dbHost)
+		redisClient, err := db.NewRedisClient(dbHost, db.Password(dbPassword))
 		if err != nil {
 			panic(err)
 		}
@@ -48,7 +49,7 @@ var internalCmd = &cobra.Command{
 		middlewares = append(middlewares, md.DB(redisClient))
 		middlewares = append(middlewares, md.AWSSession(s3Region, s3Endpoint, s3DisableSSL))
 		middlewares = append(middlewares, md.Cors())
-		middlewares = append(middlewares, md.NewWorker(dbHost, workerProducerName, workerQueueName))
+		middlewares = append(middlewares, md.NewWorker(redisClient, workerProducerName, workerQueueName))
 
 		i, err := internal.NewInternalAPI(middlewares...)
 		if err != nil {
@@ -68,13 +69,15 @@ func init() {
 
 	f.String(addressInternalFlag, ":8081", "server address")
 	f.String(dbHostInternalFlag, "127.0.0.1:6379", "database host")
+	f.String(dbPasswordInternalFlag, "", "database password")
 	f.String(s3RegionFlag, "eu-west-1", "s3 region")
 	f.String(s3EndpointFlag, "localhost:4572", "s3 endpoint")
 	f.Bool(s3DisableSSLFlag, true, "disable SSL verification for s3")
 	f.Bool(logDebugFlag, false, "sets log level to debug")
 
 	viper.BindEnv(addressInternalFlag, "ADDRESS_HOST")
 	viper.BindEnv(dbHostInternalFlag, "DB_HOST")
+	viper.BindEnv(dbPasswordInternalFlag, "DB_PASSWORD")
 	viper.BindEnv(s3RegionFlag, "S3_REGION")
 	viper.BindEnv(s3EndpointFlag, "S3_ENDPOINT")
 	viper.BindEnv(s3DisableSSLFlag, "S3_DISABLE_SSL")

diff --git a/cmd/public.go b/cmd/public.go
@@ -39,6 +39,7 @@ APIs for serving the personalized content.`,
 		// read parameters in input
 		addr := viper.GetString(addressPublicFlag)
 		dbHost := viper.GetString(dbHostPublicFlag)
+		dbPassword := viper.GetString(dbPasswordPublicFlag)
 		logType := viper.GetString(recommendationLogsFlag)
 		logDebug := viper.GetBool(logDebugFlag)
 
@@ -49,7 +50,7 @@ APIs for serving the personalized content.`,
 		}
 
 		// instantiate Redis client
-		redisClient, err := db.NewRedisClient(dbHost)
+		redisClient, err := db.NewRedisClient(dbHost, db.Password(dbPassword))
 		if err != nil {
 			panic(err)
 		}
@@ -107,6 +108,7 @@ func init() {
 	// mandatory parameters
 	f.StringP(addressPublicFlag, "a", ":8082", "server address")
 	f.StringP(dbHostPublicFlag, "d", "127.0.0.1:6379", "database host")
+	f.String(dbPasswordPublicFlag, "", "database password")
 	f.Bool(logDebugFlag, false, "sets log level to debug")
 
 	// optional parameters
@@ -121,6 +123,7 @@ func init() {
 
 	viper.BindEnv(addressPublicFlag, "ADDRESS_HOST")
 	viper.BindEnv(dbHostPublicFlag, "DB_HOST")
+	viper.BindEnv(dbPasswordPublicFlag, "DB_PASSWORD")
 	viper.BindEnv(logDebugFlag, "LOG_DEBUG")
 	viper.BindEnv(recommendationLogsFlag, "REC_LOGS_TYPE")
 	viper.BindEnv(recommendationKafkaBrokersFlag, "REC_LOGS_BROKERS")

diff --git a/cmd/root.go b/cmd/root.go
@@ -7,12 +7,15 @@ import (
 )
 
 var (
-	addressPublicFlag   = "address-host-public"
-	dbHostPublicFlag    = "db-host-public"
-	addressInternalFlag = "address-host-internal"
-	dbHostInternalFlag  = "db-host-internal"
-	workerBrokerFlag    = "worker-broker-url"
-	logDebugFlag        = "debug"
+	addressPublicFlag      = "address-host-public"
+	dbHostPublicFlag       = "db-host-public"
+	dbPasswordPublicFlag   = "db-password-public"
+	addressInternalFlag    = "address-host-internal"
+	dbHostInternalFlag     = "db-host-internal"
+	dbPasswordInternalFlag = "db-password-internal"
+	workerBrokerFlag       = "worker-broker-url"
+	workerPasswordFlag     = "worker-password"
+	logDebugFlag           = "debug"
 )
 
 // rootCmd represents the base command when called without any subcommands

diff --git a/cmd/worker.go b/cmd/worker.go
@@ -27,22 +27,23 @@ var workerCmd = &cobra.Command{
 	that will be executed one they arrive.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		brokerWorker := viper.GetString(workerBrokerFlag)
+		passwordWorker := viper.GetString(workerPasswordFlag)
 
 		// instantiate Redis client
-		redisClient, err := db.NewRedisClient(brokerWorker)
+		rc, err := db.NewRedisClient(brokerWorker, db.Password(passwordWorker))
 		if err != nil {
 			panic(err)
 		}
 
-		l, err := redisClient.Lock(worker.WorkerLockKey)
-		defer redisClient.Unlock(worker.WorkerLockKey)
+		l, err := rc.Lock(worker.WorkerLockKey)
+		defer rc.Unlock(worker.WorkerLockKey)
 
 		if l == false || err != nil {
 			log.Error().Err(err).Msg("REDIS failed to acquire lock")
 			os.Exit(0)
 		}
 
-		w, err := worker.New(brokerWorker, workerConsumerName, workerQueueName)
+		w, err := worker.New(rc.Client, workerConsumerName, workerQueueName)
 		if err != nil {
 			panic(err)
 		}
@@ -61,7 +62,7 @@ var workerCmd = &cobra.Command{
 		for {
 			select {
 			case <-ticker.C:
-				if err := redisClient.ExtendTTL(worker.WorkerLockKey); err != nil {
+				if err := rc.ExtendTTL(worker.WorkerLockKey); err != nil {
 					log.Error().Msg(err.Error())
 					w.Close()
 					break EXITLOOP
@@ -82,8 +83,10 @@ func init() {
 	f := workerCmd.PersistentFlags()
 
 	f.String(workerBrokerFlag, "127.0.0.1:6379", "broker url for the workers")
+	f.String(workerPasswordFlag, "", "broker password")
 
 	viper.BindEnv(workerBrokerFlag, "WORKER_BROKER_URL")
+	viper.BindEnv(workerPasswordFlag, "WORKER_PASSWORD")
 
 	viper.BindPFlags(f)
 }
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -21,6 +21,7 @@ services:
     restart: always
     command:
       - "redis-server"
+      # - "--requirepass qwerty"
       - "--appendonly yes"
     ports:
       - "6379:6379"
diff --git a/go.mod b/go.mod
@@ -4,47 +4,30 @@ go 1.13
 
 require (
 	github.com/Shopify/sarama v1.24.0
-	github.com/adjust/gocheck v0.0.0-20131111155431-fbc315b36e0e // indirect
-	github.com/adjust/rmq v1.0.0
-	github.com/adjust/uniuri v0.0.0-20130923163420-498743145e60 // indirect
+	github.com/adjust/rmq/v3 v3.0.0
 	github.com/allegro/bigcache v1.2.1
 	github.com/aws/aws-sdk-go v1.25.8
 	github.com/banzaicloud/go-gin-prometheus v0.0.0-20190417120951-df9373ad5327
-	github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b // indirect
 	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.4.1
-	github.com/garyburd/redigo v1.6.0 // indirect
 	github.com/gin-contrib/cors v1.3.0
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/gin-gonic/gin v1.4.0
 	github.com/go-playground/locales v0.12.1 // indirect
 	github.com/go-playground/universal-translator v0.16.0 // indirect
-	github.com/go-redis/redis v6.15.6+incompatible
+	github.com/go-redis/redis/v7 v7.4.0
 	github.com/google/uuid v1.1.1
-	github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 // indirect
 	github.com/json-iterator/go v1.1.7
 	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/leodido/go-urn v1.1.0 // indirect
-	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/mattn/go-isatty v0.0.8 // indirect
-	github.com/onsi/ginkgo v1.10.0 // indirect
-	github.com/onsi/gomega v1.7.0 // indirect
 	github.com/prometheus/client_golang v1.2.1
 	github.com/rs/zerolog v1.15.0
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.4.0
-	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
-	github.com/stretchr/testify v1.4.0
-	github.com/tsenart/vegeta v12.7.0+incompatible
+	github.com/stretchr/testify v1.5.1
 	github.com/ugorji/go v1.1.7 // indirect
 	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect
-	golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect
 	golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
-	golang.org/x/text v0.3.2 // indirect
-	gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a // indirect
 	gopkg.in/go-playground/validator.v9 v9.29.1
-	gopkg.in/redis.v3 v3.6.4 // indirect
-	gopkg.in/yaml.v2 v2.2.4 // indirect
-	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
 )