Remove cyclic dependency and allow to pass Parallelism as an argument (…

…#1859) * Remove cyclic dependency from package kopia Signed-off-by: Rajat Gupta <[email protected]> * Update Tests Signed-off-by: Rajat Gupta <[email protected]> * Change Export Variables to Non Export type Signed-off-by: Rajat Gupta <[email protected]> Signed-off-by: Rajat Gupta <[email protected]>
kanisterio · Jan 12, 2023 · e65900e · e65900e
1 parent 1602d1b
commit e65900e
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 42 deletions.
diff --git a/pkg/kopia/command/helpers.go b/pkg/kopia/command/helpers.go
@@ -15,26 +15,48 @@
 package command
 
 import (
-	"github.com/kanisterio/kanister/pkg/kopia"
 	"github.com/kanisterio/kanister/pkg/logsafe"
 	"github.com/kanisterio/kanister/pkg/utils"
 )
 
+const (
+	// dataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls
+	// kopia content cache size for general command workloads
+	dataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB"
+	// defaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads
+	defaultDataStoreGeneralMetadataCacheSizeMB = 500
+	// dataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls
+	// kopia metadata cache size for general command workloads
+	dataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB"
+	// defaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads
+	defaultDataStoreRestoreContentCacheSizeMB = 500
+	// defaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads
+	defaultDataStoreGeneralContentCacheSizeMB = 0
+	// dataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls
+	// kopia content cache size for restore workloads
+	dataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB"
+	// defaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads
+	defaultDataStoreRestoreMetadataCacheSizeMB = 500
+	// dataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls
+	// kopia metadata cache size for restore workloads
+	dataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB"
+)
+
 type policyChanges map[string]string
 
 // GetCacheSizeSettingsForSnapshot returns the feature setting cache size values to be used
 // for initializing repositories that will be performing general command workloads that benefit from
 // cacheing metadata only.
 func GetCacheSizeSettingsForSnapshot() (contentCacheMB, metadataCacheMB int) {
-	return utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralContentCacheSizeMBVarName, kopia.DefaultDataStoreGeneralContentCacheSizeMB),
-		utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralMetadataCacheSizeMBVarName, kopia.DefaultDataStoreGeneralMetadataCacheSizeMB)
+	return utils.GetEnvAsIntOrDefault(dataStoreGeneralContentCacheSizeMBVarName, defaultDataStoreGeneralContentCacheSizeMB),
+		utils.GetEnvAsIntOrDefault(dataStoreGeneralMetadataCacheSizeMBVarName, defaultDataStoreGeneralMetadataCacheSizeMB)
 }
 
 // GetCacheSizeSettingsForRestore returns the feature setting cache size values to be used
 // for initializing repositories that will be performing restore workloads
 func GetCacheSizeSettingsForRestore() (contentCacheMB, metadataCacheMB int) {
-	return utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreContentCacheSizeMBVarName, kopia.DefaultDataStoreRestoreContentCacheSizeMB),
-		utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreMetadataCacheSizeMBVarName, kopia.DefaultDataStoreRestoreMetadataCacheSizeMB)
+	return utils.GetEnvAsIntOrDefault(dataStoreRestoreContentCacheSizeMBVarName, defaultDataStoreRestoreContentCacheSizeMB),
+		utils.GetEnvAsIntOrDefault(dataStoreRestoreMetadataCacheSizeMBVarName, defaultDataStoreRestoreMetadataCacheSizeMB)
 }
 
 type GeneralCommandArgs struct {

diff --git a/pkg/kopia/command/snapshot.go b/pkg/kopia/command/snapshot.go
@@ -18,24 +18,25 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kanisterio/kanister/pkg/kopia"
 	"github.com/kanisterio/kanister/pkg/utils"
 )
 
 const (
-	requireLogLevelInfo = true
+	requireLogLevelInfo        = true
+	manifestTypeSnapshotFilter = "type:snapshot"
 )
 
 type SnapshotCreateCommandArgs struct {
 	*CommandArgs
 	PathToBackup           string
 	Tags                   []string
 	ProgressUpdateInterval time.Duration
+	Parallelism            int
 }
 
 // SnapshotCreate returns the kopia command for creation of a snapshot
 func SnapshotCreate(cmdArgs SnapshotCreateCommandArgs) []string {
-	parallelismStr := strconv.Itoa(utils.GetEnvAsIntOrDefault(kopia.DataStoreParallelUploadVarName, kopia.DefaultDataStoreParallelUpload))
+	parallelismStr := strconv.Itoa(cmdArgs.Parallelism)
 	args := commonArgs(cmdArgs.CommandArgs, requireLogLevelInfo)
 	args = args.AppendLoggable(snapshotSubCommand, createSubCommand, cmdArgs.PathToBackup, jsonFlag)
 	args = args.AppendLoggableKV(parallelFlag, parallelismStr)
@@ -145,7 +146,7 @@ type SnapListAllWithSnapIDsCommandArgs struct {
 func SnapListAllWithSnapIDs(cmdArgs SnapListAllWithSnapIDsCommandArgs) []string {
 	args := commonArgs(cmdArgs.CommandArgs, false)
 	args = args.AppendLoggable(manifestSubCommand, listSubCommand, jsonFlag)
-	args = args.AppendLoggableKV(filterFlag, kopia.ManifestTypeSnapshotFilter)
+	args = args.AppendLoggableKV(filterFlag, manifestTypeSnapshotFilter)
 
 	return stringSliceCommand(args)
 }

diff --git a/pkg/kopia/command/snapshot_test.go b/pkg/kopia/command/snapshot_test.go
@@ -42,6 +42,7 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *C) {
 					CommandArgs:            commandArgs,
 					PathToBackup:           "path/to/backup",
 					ProgressUpdateInterval: 0,
+					Parallelism:            8,
 				}
 				return SnapshotCreate(args)
 			},
@@ -53,6 +54,7 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *C) {
 					CommandArgs:            commandArgs,
 					PathToBackup:           "path/to/backup",
 					ProgressUpdateInterval: 1*time.Minute + 35*time.Second,
+					Parallelism:            8,
 				}
 				return SnapshotCreate(args)
 			},

diff --git a/pkg/kopia/const.go b/pkg/kopia/const.go
@@ -20,37 +20,4 @@ const (
 
 	// DefaultClientCacheDirectory is the directory where kopia content cache is created
 	DefaultClientCacheDirectory = "/tmp/kopia-cache"
-
-	// DefaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads
-	DefaultDataStoreGeneralContentCacheSizeMB = 0
-	// DataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls
-	// kopia content cache size for general command workloads
-	DataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB"
-
-	// DefaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads
-	DefaultDataStoreGeneralMetadataCacheSizeMB = 500
-	// DataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls
-	// kopia metadata cache size for general command workloads
-	DataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB"
-
-	// DefaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads
-	DefaultDataStoreRestoreContentCacheSizeMB = 500
-	// DataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls
-	// kopia content cache size for restore workloads
-	DataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB"
-
-	// DefaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads
-	DefaultDataStoreRestoreMetadataCacheSizeMB = 500
-	// DataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls
-	// kopia metadata cache size for restore workloads
-	DataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB"
-
-	// DefaultDataStoreParallelUpload is the default value for data store parallelism
-	DefaultDataStoreParallelUpload = 8
-
-	// DataStoreParallelUploadVarName is the name of the environment variable that controls
-	// kopia parallelism during snapshot create commands
-	DataStoreParallelUploadVarName = "DATA_STORE_PARALLEL_UPLOAD"
-
-	ManifestTypeSnapshotFilter = "type:snapshot"
 )