diff --git a/pkg/kopia/command/helpers.go b/pkg/kopia/command/helpers.go index a09f1c3850..9553e68c28 100644 --- a/pkg/kopia/command/helpers.go +++ b/pkg/kopia/command/helpers.go @@ -15,26 +15,48 @@ package command import ( - "github.com/kanisterio/kanister/pkg/kopia" "github.com/kanisterio/kanister/pkg/logsafe" "github.com/kanisterio/kanister/pkg/utils" ) +const ( + // dataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls + // kopia content cache size for general command workloads + dataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB" + // defaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads + defaultDataStoreGeneralMetadataCacheSizeMB = 500 + // dataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls + // kopia metadata cache size for general command workloads + dataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB" + // defaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads + defaultDataStoreRestoreContentCacheSizeMB = 500 + // defaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads + defaultDataStoreGeneralContentCacheSizeMB = 0 + // dataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls + // kopia content cache size for restore workloads + dataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB" + // defaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads + defaultDataStoreRestoreMetadataCacheSizeMB = 500 + // dataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls + // kopia metadata cache size for restore workloads + dataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB" +) + type policyChanges map[string]string // GetCacheSizeSettingsForSnapshot returns the feature setting cache size values to be used // for initializing repositories that will be performing general command workloads that benefit from // cacheing metadata only. func GetCacheSizeSettingsForSnapshot() (contentCacheMB, metadataCacheMB int) { - return utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralContentCacheSizeMBVarName, kopia.DefaultDataStoreGeneralContentCacheSizeMB), - utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralMetadataCacheSizeMBVarName, kopia.DefaultDataStoreGeneralMetadataCacheSizeMB) + return utils.GetEnvAsIntOrDefault(dataStoreGeneralContentCacheSizeMBVarName, defaultDataStoreGeneralContentCacheSizeMB), + utils.GetEnvAsIntOrDefault(dataStoreGeneralMetadataCacheSizeMBVarName, defaultDataStoreGeneralMetadataCacheSizeMB) } // GetCacheSizeSettingsForRestore returns the feature setting cache size values to be used // for initializing repositories that will be performing restore workloads func GetCacheSizeSettingsForRestore() (contentCacheMB, metadataCacheMB int) { - return utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreContentCacheSizeMBVarName, kopia.DefaultDataStoreRestoreContentCacheSizeMB), - utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreMetadataCacheSizeMBVarName, kopia.DefaultDataStoreRestoreMetadataCacheSizeMB) + return utils.GetEnvAsIntOrDefault(dataStoreRestoreContentCacheSizeMBVarName, defaultDataStoreRestoreContentCacheSizeMB), + utils.GetEnvAsIntOrDefault(dataStoreRestoreMetadataCacheSizeMBVarName, defaultDataStoreRestoreMetadataCacheSizeMB) } type GeneralCommandArgs struct { diff --git a/pkg/kopia/command/snapshot.go b/pkg/kopia/command/snapshot.go index eac44da402..00d63eacca 100644 --- a/pkg/kopia/command/snapshot.go +++ b/pkg/kopia/command/snapshot.go @@ -18,12 +18,12 @@ import ( "strconv" "time" - "github.com/kanisterio/kanister/pkg/kopia" "github.com/kanisterio/kanister/pkg/utils" ) const ( - requireLogLevelInfo = true + requireLogLevelInfo = true + manifestTypeSnapshotFilter = "type:snapshot" ) type SnapshotCreateCommandArgs struct { @@ -31,11 +31,12 @@ type SnapshotCreateCommandArgs struct { PathToBackup string Tags []string ProgressUpdateInterval time.Duration + Parallelism int } // SnapshotCreate returns the kopia command for creation of a snapshot func SnapshotCreate(cmdArgs SnapshotCreateCommandArgs) []string { - parallelismStr := strconv.Itoa(utils.GetEnvAsIntOrDefault(kopia.DataStoreParallelUploadVarName, kopia.DefaultDataStoreParallelUpload)) + parallelismStr := strconv.Itoa(cmdArgs.Parallelism) args := commonArgs(cmdArgs.CommandArgs, requireLogLevelInfo) args = args.AppendLoggable(snapshotSubCommand, createSubCommand, cmdArgs.PathToBackup, jsonFlag) args = args.AppendLoggableKV(parallelFlag, parallelismStr) @@ -145,7 +146,7 @@ type SnapListAllWithSnapIDsCommandArgs struct { func SnapListAllWithSnapIDs(cmdArgs SnapListAllWithSnapIDsCommandArgs) []string { args := commonArgs(cmdArgs.CommandArgs, false) args = args.AppendLoggable(manifestSubCommand, listSubCommand, jsonFlag) - args = args.AppendLoggableKV(filterFlag, kopia.ManifestTypeSnapshotFilter) + args = args.AppendLoggableKV(filterFlag, manifestTypeSnapshotFilter) return stringSliceCommand(args) } diff --git a/pkg/kopia/command/snapshot_test.go b/pkg/kopia/command/snapshot_test.go index d19837599d..9ab474e12a 100644 --- a/pkg/kopia/command/snapshot_test.go +++ b/pkg/kopia/command/snapshot_test.go @@ -42,6 +42,7 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *C) { CommandArgs: commandArgs, PathToBackup: "path/to/backup", ProgressUpdateInterval: 0, + Parallelism: 8, } return SnapshotCreate(args) }, @@ -53,6 +54,7 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *C) { CommandArgs: commandArgs, PathToBackup: "path/to/backup", ProgressUpdateInterval: 1*time.Minute + 35*time.Second, + Parallelism: 8, } return SnapshotCreate(args) }, diff --git a/pkg/kopia/const.go b/pkg/kopia/const.go index 1d7c3adb57..a1b1fb7cbe 100644 --- a/pkg/kopia/const.go +++ b/pkg/kopia/const.go @@ -20,37 +20,4 @@ const ( // DefaultClientCacheDirectory is the directory where kopia content cache is created DefaultClientCacheDirectory = "/tmp/kopia-cache" - - // DefaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads - DefaultDataStoreGeneralContentCacheSizeMB = 0 - // DataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls - // kopia content cache size for general command workloads - DataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB" - - // DefaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads - DefaultDataStoreGeneralMetadataCacheSizeMB = 500 - // DataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls - // kopia metadata cache size for general command workloads - DataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB" - - // DefaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads - DefaultDataStoreRestoreContentCacheSizeMB = 500 - // DataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls - // kopia content cache size for restore workloads - DataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB" - - // DefaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads - DefaultDataStoreRestoreMetadataCacheSizeMB = 500 - // DataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls - // kopia metadata cache size for restore workloads - DataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB" - - // DefaultDataStoreParallelUpload is the default value for data store parallelism - DefaultDataStoreParallelUpload = 8 - - // DataStoreParallelUploadVarName is the name of the environment variable that controls - // kopia parallelism during snapshot create commands - DataStoreParallelUploadVarName = "DATA_STORE_PARALLEL_UPLOAD" - - ManifestTypeSnapshotFilter = "type:snapshot" )