diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b9c3338..0a4c370 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,12 +14,21 @@ jobs: with: go-version: 1.15 + - name: Lint + run: make lintci + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: version: latest args: release --rm-dist --snapshot --skip-publish -f build/ci/.goreleaser.yml + - name: Run GoReleaser for root version + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist --snapshot --skip-publish -f build/ci/.goreleaser_privileged.yml + test: runs-on: ubuntu-20.04 steps: @@ -41,7 +50,26 @@ jobs: - name: install redis-cli run: sudo apt-get install redis-tools - - name: Test - run: go test -v ./... + - name: install xfsprogs + run: sudo apt-get install xfsprogs + + - name: install xfsdump + run: sudo apt-get install xfsdump + + - name: create xfs filesystem + run: | + dd if=/dev/zero of=loopfile.img bs=1M count=20 + mkfs.xfs loopfile.img + sudo losetup /dev/loop10 loopfile.img + mkdir xfsmount + sudo mount /dev/loop10 xfsmount + + - name: Test xfs with sudo + run: sudo go test -v ./test/pkg/source/xfstest/... + env: + RESTIC_PASSWORD: mongorepo + + - name: Test remaining sources + run: go test -v $(go list ./test/... | grep -v github.com/mittwald/brudi/test/pkg/source/xfstest) env: - RESTIC_PASSWORD: mongorepo \ No newline at end of file + RESTIC_PASSWORD: mongorepo diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0af6987..23b54d7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,3 +17,7 @@ jobs: - run: curl -sL https://git.io/goreleaser | bash -s -- --config build/ci/.goreleaser.yml --rm-dist env: GITHUB_TOKEN: ${{ secrets.RELEASE_USER_TOKEN }} + + - run: curl -sL https://git.io/goreleaser | bash -s -- --config build/ci/.goreleaser_privileged.yml --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_USER_TOKEN }} diff --git a/Makefile b/Makefile index aac60ea..25a9858 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ lintci: docker run --rm \ -v $(CURDIR):/app \ -w /app \ - -e GOLANGCI_ADDITIONAL_YML=/app/build/package/ci/.golangci.yml \ + -e GOLANGCI_ADDITIONAL_YML=/app/build/ci/.golangci.yml \ quay.io/mittwald/golangci-lint:0.0.8 \ golangci-lint run -v --fix ./... @@ -37,7 +37,7 @@ lint: docker run --rm \ -v $(shell go env GOPATH):/go \ -v ${CURDIR}:/app -w /app \ - -e GOLANGCI_ADDITIONAL_YML=/app/build/package/ci/.golangci.yml \ + -e GOLANGCI_ADDITIONAL_YML=/app/build/ci/.golangci.yml \ quay.io/mittwald/golangci-lint:0.0.8 \ golangci-lint run -v --fix ./... diff --git a/README.md b/README.md index dfbf600..e492cac 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ In general everybody's doing some sort of `dump` or `tar` and backing up the res This is why `brudi` was born. `brudi` supports several backup-methods and is configurable by a simple `yaml` file. The advantage of `brudi` is, that you can create a backup of a source of your choice and save it with `restic` afterwards in one step. -Under the hood, `brudi` uses the given binaries like `mysqldump`, `mongodump`, `pg_dump`, `tar` or `restic`. +Under the hood, `brudi` uses the given binaries like `mysqldump`, `mongodump`, `pg_dump`, `tar`, `xfsdump` or `restic`. Using `brudi` will save you from finding yourself writing bash-scripts to create your backups. @@ -13,33 +13,35 @@ Besides creating backups, `brudi` can also be used to restore your data from bac ## Table of contents - - [Usage](#usage) - - [CLI](#cli) - - [Docker](#docker) - - [Configuration](#configuration) - - [Sources](#sources) - - [Tar](#tar) - - [MySQLDump](#mysqldump) - - [MongoDump](#mongodump) - - [PgDump](#pgdump) - - [Limitations](#limitations) - - [Redis](#redis) - - [Restic](#restic) - - [Forget](#forget) - - [Sensitive data: Environment variables](#sensitive-data-environment-variables) - - [Gzip support for binaries without native gzip support](#gzip-support-for-binaries-without-native-gzip-support) - - [Restoring from backup](#restoring-from-backup) - - [TarRestore](#tarrestore) - - [MongoRestore](#mongorestore) - - [MySQLRestore](#mysqlrestore) - - [PgRestore](#pgrestore) - - [Restore using pg_restore](#restore-using-pg_restore) - - [Restore using psql](#restore-using-psql) - - [Restoring using restic](#restoring-using-restic) - - [Featurestate](#featurestate) - - [Source backup methods](#source-backup-methods) - - [Restore backup methods](#restore-backup-methods) - - [Incremental backup of the source backups](#incremental-backup-of-the-source-backups) +- [Usage](#usage) + - [CLI](#cli) + - [Docker](#docker) + - [Configuration](#configuration) + - [Sources](#sources) + - [Tar](#tar) + - [MySQLDump](#mysqldump) + - [MongoDump](#mongodump) + - [PgDump](#pgdump) + - [Limitations](#limitations) + - [Redis](#redis) + - [XFSdump](#xfsdump) + - [Restic](#restic) + - [Forget](#forget) + - [Sensitive data: Environment variables](#sensitive-data-environment-variables) + - [Gzip support for binaries without native gzip support](#gzip-support-for-binaries-without-native-gzip-support) + - [Restoring from backup](#restoring-from-backup) + - [TarRestore](#tarrestore) + - [MongoRestore](#mongorestore) + - [MySQLRestore](#mysqlrestore) + - [PgRestore](#pgrestore) + - [Restore using pg_restore](#restore-using-pg_restore) + - [Restore using psql](#restore-using-psql) + - [XFSRestore](#xfsrestore) + - [Restoring using restic](#restoring-using-restic) +- [Featurestate](#featurestate) + - [Source backup methods](#source-backup-methods) + - [Restore backup methods](#restore-backup-methods) + - [Incremental backup of the source backups](#incremental-backup-of-the-source-backups) ## Usage @@ -51,9 +53,9 @@ In order to use the `brudi`-binary on your local machine or a remote server of y - `mysqldump` (required when running `brudi mysqldump`) - `tar` (required when running `brudi tar`) - `redis-cli` (required when running `brudi redisdump`) +- `xfsdump` (required when running `brudi xfsdump`) - `restic` (required when running `brudi --restic`) - ```shell $ brudi --help @@ -75,6 +77,8 @@ Available Commands: redisdump Creates an rdb dump of your desired server tar Creates a tar archive of your desired tarrestore Restores files from a tar archive + xfsdump Creates a dump of your desired xfs filesystem + xfsrestore Restores a dump of an xfs filesystem version Print the version number of brudi Flags: @@ -92,9 +96,33 @@ Use "brudi [command] --help" for more information about a command. In case you don't want to install additional tools, you can also use `brudi` inside docker: -`docker run --rm -v ${HOME}/.brudi.yml:/home/brudi/.brudi.yml quay.io/mittwald/brudi mongodump --restic --cleanup` +`docker run --rm -v ${HOME}/.brudi.yaml:/home/brudi/.brudi.yaml quay.io/mittwald/brudi mongodump --restic --cleanup` + +The docker-image comes with all required binaries, except for `xfsdump` and `xfsrestore`. Since usage of `xfsdump` requires root access, +a separate docker image is provided, as detailed below. + +WARNING: The following image supports `xfsdump` and thus requires to be run in a privileged container, and the image itself runs as root. Only use this if you are fully aware of what you are doing and absolutely need `xfsdump` capabilities from the docker image + +An example docker run command could look like this: + +`sudo docker run --privileged --rm --mount 'type=bind,src=${HOME}.brudi.xfsdump.yaml,dst=/root/.brudi.yaml' -v ${HOME}examplemount:/home/examplemount -v ${HOME}example_backup_location:/home/example_backup_location quay.io/mittwald/brudi_root xfsdump` + + +This command will run the `brudi_root` image in a privileged container, with an Ubuntu running as root, mount your config file, mount your local filesystem mounted at `/home/examplemount` into the container, mount a directory to save the backup in and finally run `brudi xfsdump` +A matching config would look like this: + +```yaml +xfsdump: + options: + flags: + level: 0 + dontPromptOperator: true + destination: /home/example_backup_location/test.xfsdump + additionalArgs: [] + targetFS: /home/examplemount +``` -The docker-image comes with all required binaries. +We highly recommend you use the normal `brudi` image, unless you absolutely need the xfsdump capability. ### Configuration @@ -116,13 +144,14 @@ Therefore you can simply refer to the official documentation for explanations on - [`mongodump`](https://docs.mongodb.com/manual/reference/program/mongodump/#options) - [`mysqldump`](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html#mysqldump-option-summary) - [`pg_dump`](https://www.postgresql.org/docs/12/app-pgdump.html) +- [`xfsdump`](https://man7.org/linux/man-pages/man8/xfsdump.8.html) Every source has a an `additionalArgs`-key which's value is an array of strings. The value of this key is appended to the command, generated by `brudi`. Even though `brudi` should support all cli-flags to be configured via the `.yaml`-file, there may be flags which are not. In this case, use the `additionalArgs`-key. It is also possible to provide more than one configuration file, for example `-c mongodump.yaml -c restic.yaml`. These configs get merged at runtime. -If available, the default config will always be laoded first and then overwritten with any values from user-specified files. +If available, the default config will always be laoded first and then overwritten with any values from user-specified files. In case the same config file has been provided more than once, only the first instance will be taken into account. #### Sources @@ -243,6 +272,28 @@ Becomes the following command: As `redis-cli` is not a dedicated backup tool but a client for `redis`, only a limited number of flags are available by default, as you can see [here](pkg/source/redisdump/cli.go#L7). +##### XFSdump + +Please be aware that `xfsdump` requires root privileges in order to work + +```yaml +xfsdump: + options: + flags: + level: 0 + destination: test.xfsdump + dontPromptOperator: true + additionalArgs: [] + targetFS: /testmount +``` + +Running: `brudi xfsdump -c ${HOME}/.brudi.yml` + +Becomes the following command: +`xfsdump -f test.xfsdump -F -l 0 /testmount` + +All available flags to be set in the `.yaml`-configuration can be found [here](pkg/source/xfsdump/cli.go#L7). + #### Restic In case you're running your backup with the `--restic`-flag, you need to provide a [valid configuration for restic](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html). @@ -340,7 +391,6 @@ mysqlrestore: sourceFile: /tmp/test.sqldump.gz ``` - #### Restoring from backup ##### TarRestore @@ -360,7 +410,7 @@ tarrestore: Running: `brudi tarrestore -c ${HOME}/.brudi.yml` Becomes the following command: -`tar -x -z -f /tmp/test.tar.gz -C /` +`tar -x -z -f /tmp/test.tar.gz -C /` ##### MongoRestore @@ -376,12 +426,12 @@ Becomes the following command: archive: /tmp/dump.tar.gz additionalArgs: [] ``` - - Running: `brudi mongorestore -c ${HOME}/.brudi.yml ` - + + Running: `brudi mongorestore -c ${HOME}/.brudi.yml` + Becomes the following command: `mongorestore --host=127.0.0.1 --port=27017 --username=root --password=mongodbroot --gzip --archive=/tmp/dump.tar.gz` - + All available flags to be set in the `.yaml`-configuration can be found [here](pkg/source/mongorestore/cli.go#L7). ##### MySQLRestore @@ -455,17 +505,37 @@ psql: Running: `brudi pgrestore -c ${HOME}/.brudi.yml` -Becomes the following command: +Becomes the following command: `psql --host=127.0.0.1 --port=5432 --user=postgresuser --db-name=postgres < /tmp/postgress.dump` This command has to be used if the `format` option was set to `plain` in `pg_dump`, which is the default. All available flags to be set in the `.yaml`-configuration can be found [here](pkg/source/psql/cli.go#L7). +###### XFSrestore + +```yaml +xfsrestore: + options: + flags: + source: test.xfsdump + inhibitInteractivePrompts: true + additionalArgs: [] + destFS: /testmount +``` + +Running: `brudi xfsrestore -c ${HOME}/.brudi.yml` + +Becomes the following command: +`xfsrestore -f test.xfsdump -F -l 0 /testmount` + +All available flags to be set in the `.yaml`-configuration can be found [here](pkg/source/xfsrestore/cli.go#L7). + ##### Restoring using restic -Backups can be pulled from a `restic` repository and applied to your server by using the `--restic` flag in your brudi command. +Backups can be pulled from a `restic` repository and applied to your server by using the `--restic` flag in your brudi command. Example configuration for `mongorestore`: + ```yaml mongorestore: options: @@ -488,7 +558,7 @@ restic: ``` This will pull the latest snapshot of `/tmp/dump.tar.gz` from the repository, which `mongorestore` then uses to restore the server. -It is also possible to specify concrete snapshot-ids instead of `latest`. +It is also possible to specify concrete snapshot-ids instead of `latest`. ## Featurestate @@ -502,12 +572,12 @@ It is also possible to specify concrete snapshot-ids instead of `latest`. ### Restore backup methods -- [x] `mysqlrestore` +- [x] `mysqlrestore` - [x] `mongorestore` - [x] `tarrestore` - [x] `pgrestore` - [ ] `redisrestore` - + ### Incremental backup of the source backups - [x] `restic` diff --git a/build/ci/.golangci.yml b/build/ci/.golangci.yml index f69bcd4..2d37021 100644 --- a/build/ci/.golangci.yml +++ b/build/ci/.golangci.yml @@ -3,5 +3,6 @@ run: skip-dirs: - test/ + - dist/ timeout: 10m diff --git a/build/ci/.goreleaser.yml b/build/ci/.goreleaser.yml index c010ac1..645c9f6 100644 --- a/build/ci/.goreleaser.yml +++ b/build/ci/.goreleaser.yml @@ -1,7 +1,7 @@ before: hooks: - go mod download - - make lintci + builds: - env: diff --git a/build/ci/.goreleaser_privileged.yml b/build/ci/.goreleaser_privileged.yml new file mode 100644 index 0000000..35de274 --- /dev/null +++ b/build/ci/.goreleaser_privileged.yml @@ -0,0 +1,49 @@ +before: + hooks: + - go mod download + +builds: + - + env: + - CGO_ENABLED=0 + binary: brudi + ldflags: + - -s + - -w + - -X 'github.com/mittwald/brudi/cmd.tag={{ .Tag }}' + goos: + - darwin + - linux + goarch: + - amd64 +archives: + - replacements: + darwin: Darwin + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' +dockers: + - + image_templates: + - quay.io/mittwald/brudi_root:latest + - quay.io/mittwald/brudi_root:{{ .Major }} + - quay.io/mittwald/brudi_root:{{ .Major }}.{{ .Minor }} + - quay.io/mittwald/brudi_root:{{ .Tag }} + - quay.io/mittwald/brudi_root:stable + binaries: + - brudi + dockerfile: build/docker/Dockerfile_privileged + goos: linux + goarch: amd64 + goarm: '' \ No newline at end of file diff --git a/build/docker/Dockerfile_privileged b/build/docker/Dockerfile_privileged new file mode 100644 index 0000000..cebc375 --- /dev/null +++ b/build/docker/Dockerfile_privileged @@ -0,0 +1,28 @@ +FROM ubuntu:18.04 + +LABEL maintainer="Mittwald CM Service " + +ENV BRUDI_USER="brudi" \ + BRUDI_GID="1000" \ + BRUDI_UID="1000" + +COPY brudi /usr/local/bin/brudi + +COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic +COPY --from=redis:alpine /usr/local/bin/redis-cli /usr/local/bin/redis-cli + +RUN apt-get update \ + && apt-get install -y wget gnupg \ + && wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \ + && echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list \ + && apt-get update \ + && \ + apt-get -y install \ + mongodb-org-tools \ + mysql-client \ + postgresql-client \ + xfsdump + +USER root + +ENTRYPOINT ["brudi"] \ No newline at end of file diff --git a/cmd/xfsdump.go b/cmd/xfsdump.go new file mode 100644 index 0000000..425a942 --- /dev/null +++ b/cmd/xfsdump.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "context" + + "github.com/mittwald/brudi/pkg/source" + + "github.com/spf13/cobra" + + "github.com/mittwald/brudi/pkg/source/xfsdump" +) + +var ( + xfsDumpCmd = &cobra.Command{ + Use: "xfsdump", + Short: "Creates a xfsdump of your desired file system", + Long: "Backups a given filesystem with given arguments", + Run: func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := source.DoBackupForKind(ctx, xfsdump.Kind, cleanup, useRestic, useResticForget) + if err != nil { + panic(err) + } + }, + } +) + +func init() { + rootCmd.AddCommand(xfsDumpCmd) +} diff --git a/cmd/xfsrestore.go b/cmd/xfsrestore.go new file mode 100644 index 0000000..78ecd03 --- /dev/null +++ b/cmd/xfsrestore.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "context" + + "github.com/mittwald/brudi/pkg/source" + "github.com/mittwald/brudi/pkg/source/xfsrestore" + + "github.com/spf13/cobra" +) + +var ( + xfsRestoreCmd = &cobra.Command{ + Use: "xfsrestore", + Short: "Restores a xfsdump of A file system", + Long: "Restores a given filesystem with given arguments", + Run: func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := source.DoRestoreForKind(ctx, xfsrestore.Kind, cleanup, useRestic, useResticForget) + if err != nil { + panic(err) + } + }, + } +) + +func init() { + rootCmd.AddCommand(xfsRestoreCmd) +} diff --git a/example/config/.brudi.xfsdump.yaml b/example/config/.brudi.xfsdump.yaml new file mode 100644 index 0000000..00541ae --- /dev/null +++ b/example/config/.brudi.xfsdump.yaml @@ -0,0 +1,8 @@ +xfsdump: + options: + flags: + level: 0 + dontPromptOperator: true + destination: test.xfsdump + additionalArgs: [] + targetFS: /testmount diff --git a/example/config/.brudi.xfsrestore.yaml b/example/config/.brudi.xfsrestore.yaml new file mode 100644 index 0000000..814634e --- /dev/null +++ b/example/config/.brudi.xfsrestore.yaml @@ -0,0 +1,7 @@ +xfsrestore: + options: + flags: + source: test.xfsdump + dontPrompOperator: false + additionalArgs: [] + DestFS: /testmount diff --git a/go.mod b/go.mod index e085c38..e951d0f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/go-redis/redis v6.15.9+incompatible github.com/go-sql-driver/mysql v1.5.0 github.com/gofrs/uuid v3.3.0+incompatible // indirect - github.com/google/uuid v1.1.2 github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jackc/pgx v3.6.2+incompatible github.com/lib/pq v1.9.0 // indirect diff --git a/pkg/source/backup.go b/pkg/source/backup.go index 1915384..8c73df0 100644 --- a/pkg/source/backup.go +++ b/pkg/source/backup.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/mittwald/brudi/pkg/restic" - "github.com/mittwald/brudi/pkg/source/pgdump" + "github.com/mittwald/brudi/pkg/source/xfsdump" "github.com/mittwald/brudi/pkg/source/tar" @@ -29,6 +29,8 @@ func getGenericBackendForKind(kind string) (Generic, error) { return redisdump.NewConfigBasedBackend() case tar.Kind: return tar.NewConfigBasedBackend() + case xfsdump.Kind: + return xfsdump.NewConfigBasedBackend() default: return nil, fmt.Errorf("unsupported kind '%s'", kind) } diff --git a/pkg/source/restore.go b/pkg/source/restore.go index e415f81..a6c1023 100644 --- a/pkg/source/restore.go +++ b/pkg/source/restore.go @@ -12,6 +12,7 @@ import ( "github.com/mittwald/brudi/pkg/source/pgrestore" "github.com/mittwald/brudi/pkg/source/psql" "github.com/mittwald/brudi/pkg/source/tarrestore" + "github.com/mittwald/brudi/pkg/source/xfsrestore" ) func getGenericRestoreBackendForKind(kind string) (GenericRestore, error) { @@ -26,6 +27,8 @@ func getGenericRestoreBackendForKind(kind string) (GenericRestore, error) { return tarrestore.NewConfigBasedBackend() case psql.Kind: return psql.NewConfigBasedBackend() + case xfsrestore.Kind: + return xfsrestore.NewConfigBasedBackend() default: return nil, fmt.Errorf("unsupported kind '%s'", kind) } diff --git a/pkg/source/xfsdump/backend_config_based.go b/pkg/source/xfsdump/backend_config_based.go new file mode 100644 index 0000000..08ccf47 --- /dev/null +++ b/pkg/source/xfsdump/backend_config_based.go @@ -0,0 +1,58 @@ +package xfsdump + +import ( + "context" + "fmt" + "os" + + "github.com/pkg/errors" + + "github.com/mittwald/brudi/pkg/cli" +) + +type ConfigBasedBackend struct { + cfg *Config +} + +func NewConfigBasedBackend() (*ConfigBasedBackend, error) { + config := &Config{ + Options: &Options{ + Flags: &Flags{}, + AdditionalArgs: []string{}, + TargetFS: "", + }, + } + + err := config.InitFromViper() + if err != nil { + return nil, err + } + + return &ConfigBasedBackend{cfg: config}, nil +} + +func (b *ConfigBasedBackend) CreateBackup(ctx context.Context) error { + cmd := cli.CommandType{ + Binary: binary, + Args: append(cli.StructToCLI(b.cfg.Options), b.cfg.Options.TargetFS), + } + + out, err := cli.Run(ctx, cmd) + if err != nil { + return errors.WithStack(fmt.Errorf("%+v - %s", err, out)) + } + + return nil +} + +func (b *ConfigBasedBackend) GetBackupPath() string { + return b.cfg.Options.Flags.Destination +} + +func (b *ConfigBasedBackend) GetHostname() string { + return b.cfg.HostName +} + +func (b *ConfigBasedBackend) CleanUp() error { + return os.Remove(b.GetBackupPath()) +} diff --git a/pkg/source/xfsdump/cli.go b/pkg/source/xfsdump/cli.go new file mode 100644 index 0000000..019a69b --- /dev/null +++ b/pkg/source/xfsdump/cli.go @@ -0,0 +1,41 @@ +package xfsdump + +const ( + binary = "xfsdump" +) + +type Options struct { + Flags *Flags + AdditionalArgs []string + TargetFS string // filesystem to be dumped +} + +type Flags struct { + IgnoreFilesWithOfflineCopies bool `flag:"-a"` + Exclude bool `flag:"-e"` + UseMinimalTapeProtocol bool `flag:"-m"` + Overwrite bool `flag:"-o"` + DestinationIsQIC bool `flag:"-q"` + DontDumpExtendedAttributes bool `flag:"-A"` + PreEraseMedia bool `flag:"-E"` + DontPromptOperator bool `flag:"-F"` + ShowInventory bool `flag:"-I"` + InhibitInventoryUpdate bool `flag:"-J"` + ResumeInterruptedSession bool `flag:"-R"` + InhibitDialogueTimeouts bool `flag:"-T"` + BlockSizeInBytes int `flag:"-b"` + FileSize int `flag:"-d"` + Level int `flag:"-l"` + ProgressReportInterval int `flag:"-p"` + MaxIncludedFileSize int `flag:"-z"` + BufferRingLength int `flag:"-Y"` + AlertProgramName string `flag:"-c"` + Destination string `flag:"-f"` + OnlyFromPath string `flag:"-s"` + DumpTimeFromFIle string `flag:"-t"` + BaseOnSessionID string `flag:"-B"` + SessionLabel string `flag:"-L"` + MediaObjectLabel string `flag:"-M"` + OptionsFile string `flag:"-O"` + Verbosity []string `flag:"-v"` +} diff --git a/pkg/source/xfsdump/config.go b/pkg/source/xfsdump/config.go new file mode 100644 index 0000000..cb1664e --- /dev/null +++ b/pkg/source/xfsdump/config.go @@ -0,0 +1,44 @@ +package xfsdump + +import ( + "os" + + "github.com/go-playground/validator/v10" + + "github.com/pkg/errors" + + "github.com/mittwald/brudi/pkg/config" +) + +const ( + Kind = "xfsdump" +) + +type Config struct { + Options *Options + HostName string `validate:"min=1"` +} + +func (c *Config) InitFromViper() error { + err := config.InitializeStructFromViper(Kind, c) + if err != nil { + return errors.WithStack(err) + } + + if c.HostName == "" { + c.HostName, err = os.Hostname() + if err != nil { + return errors.WithStack(err) + } + } + + return config.Validate(c, configStructLevelValidation) +} + +func configStructLevelValidation(sl validator.StructLevel) { + c := sl.Current().Interface().(Config) + + if c.Options.Flags.Destination == "" { + sl.ReportError(c.Options.Flags.Destination, "destination", "Destination", "destinationRequired", "") + } +} diff --git a/pkg/source/xfsrestore/backend_config_based.go b/pkg/source/xfsrestore/backend_config_based.go new file mode 100644 index 0000000..f7a639c --- /dev/null +++ b/pkg/source/xfsrestore/backend_config_based.go @@ -0,0 +1,58 @@ +package xfsrestore + +import ( + "context" + "fmt" + "os" + + "github.com/pkg/errors" + + "github.com/mittwald/brudi/pkg/cli" +) + +type ConfigBasedBackend struct { + cfg *Config +} + +func NewConfigBasedBackend() (*ConfigBasedBackend, error) { + config := &Config{ + Options: &Options{ + Flags: &Flags{}, + AdditionalArgs: []string{}, + DestFS: "", + }, + } + + err := config.InitFromViper() + if err != nil { + return nil, err + } + + return &ConfigBasedBackend{cfg: config}, nil +} + +func (b *ConfigBasedBackend) RestoreBackup(ctx context.Context) error { + cmd := cli.CommandType{ + Binary: binary, + Args: append(cli.StructToCLI(b.cfg.Options), b.cfg.Options.DestFS), + } + + out, err := cli.Run(ctx, cmd) + if err != nil { + return errors.WithStack(fmt.Errorf("%+v - %s", err, out)) + } + + return nil +} + +func (b *ConfigBasedBackend) GetBackupPath() string { + return b.cfg.Options.Flags.Source +} + +func (b *ConfigBasedBackend) GetHostname() string { + return b.cfg.HostName +} + +func (b *ConfigBasedBackend) CleanUp() error { + return os.Remove(b.GetBackupPath()) +} diff --git a/pkg/source/xfsrestore/cli.go b/pkg/source/xfsrestore/cli.go new file mode 100644 index 0000000..2e23cdb --- /dev/null +++ b/pkg/source/xfsrestore/cli.go @@ -0,0 +1,43 @@ +package xfsrestore + +const ( + binary = "xfsrestore" +) + +type Options struct { + Flags *Flags + AdditionalArgs []string + DestFS string // filesystem to be dumped +} + +type Flags struct { + Housekeeping bool `flag:"-a"` + PreventOverride bool `flag:"-e"` + InteractiveOperation bool `flag:"-i"` + UseMinimalTapeProtocol bool `flag:"-m"` + SourceIsQIC bool `flag:"-q"` + CumulativeMode bool `flag:"-r"` + DisplayContents bool `flag:"-t"` + DontRestoreExtendedAttributes bool `flag:"-A"` + MatchOwnershipToDumpRoot bool `flag:"-B"` + RestoreDMAPI bool `flag:"-D"` + DontOverwriteNever bool `flag:"-E"` + InhibitInteractivePrompts bool `flag:"-F"` + ShowInventory bool `flag:"-I"` + InhibitInventoryUpdate bool `flag:"-J"` + ForceCompletion bool `flag:"-Q"` + ResumeInterruptedSession bool `flag:"-R"` + InhibitDialogueTimeouts bool `flag:"-T"` + BlockSizeInBytes int `flag:"-b"` + ProgressReportInterval int `flag:"-p"` + BufferRingLength int `flag:"-Y"` + AlertProgramName string `flag:"-c"` + Source string `flag:"-f"` + RestoreOnlyNeverTHan string `flag:"-n"` + Subtree string `flag:"-s"` + Verbosity []string `flag:"-v"` + SessionLabel string `flag:"-L"` + OptionsFile string `flag:"-O"` + SessionUUID string `flag:"-S"` + Exclude string `flag:"-X"` +} diff --git a/pkg/source/xfsrestore/config.go b/pkg/source/xfsrestore/config.go new file mode 100644 index 0000000..7396e65 --- /dev/null +++ b/pkg/source/xfsrestore/config.go @@ -0,0 +1,44 @@ +package xfsrestore + +import ( + "os" + + "github.com/go-playground/validator/v10" + + "github.com/pkg/errors" + + "github.com/mittwald/brudi/pkg/config" +) + +const ( + Kind = "xfsrestore" +) + +type Config struct { + Options *Options + HostName string `validate:"min=1"` +} + +func (c *Config) InitFromViper() error { + err := config.InitializeStructFromViper(Kind, c) + if err != nil { + return errors.WithStack(err) + } + + if c.HostName == "" { + c.HostName, err = os.Hostname() + if err != nil { + return errors.WithStack(err) + } + } + + return config.Validate(c, configStructLevelValidation) +} + +func configStructLevelValidation(sl validator.StructLevel) { + c := sl.Current().Interface().(Config) + + if c.Options.Flags.Source == "" { + sl.ReportError(c.Options.Flags.Source, "source", "Source", "sourceRequired", "") + } +} diff --git a/test/pkg/source/xfstest/xfs_test.go b/test/pkg/source/xfstest/xfs_test.go new file mode 100644 index 0000000..19e361a --- /dev/null +++ b/test/pkg/source/xfstest/xfs_test.go @@ -0,0 +1,134 @@ +package xfs_test + +import ( + "bytes" + "context" + "fmt" + "os" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" + + "github.com/mittwald/brudi/pkg/source" + commons "github.com/mittwald/brudi/test/pkg/source/internal" +) + +const dumpName = "../../../../testbackup.xfsdump" +const loopDeviceName = "/dev/loop10" +const mountPoint = "../../../../xfsmount" + +type XFSTestSuite struct { + suite.Suite +} + +func (xfsTestSuite *XFSTestSuite) SetupTest() { + commons.TestSetup() +} + +// TearDownTest resets viper after a test +func (xfsTestSuite *XFSTestSuite) TearDownTest() { + viper.Reset() +} + +func (xfsTestSuite *XFSTestSuite) TestBasicXFSDump() { + ctx := context.Background() + + tarConfig := createXFSConfig("", "") + err := viper.ReadConfig(bytes.NewBuffer(tarConfig)) + xfsTestSuite.Require().NoError(err) + + dirName := fmt.Sprintf("%s/testdir", mountPoint) + + os.Mkdir(dirName, 744) + xfsTestSuite.Require().NoError(err) + + err = source.DoBackupForKind(ctx, "xfsdump", false, false, false) + xfsTestSuite.Require().NoError(err) + + err = os.Remove(dirName) + xfsTestSuite.Require().NoError(err) + + err = source.DoRestoreForKind(ctx, "xfsrestore", false, false, false) + xfsTestSuite.Require().NoError(err) + + _, err = os.Stat(dirName) + xfsTestSuite.Require().NoError(err) + + err = os.Remove(dumpName) + xfsTestSuite.Require().NoError(err) + + err = os.Remove(dirName) + xfsTestSuite.Require().NoError(err) +} + +func (xfsTestSuite *XFSTestSuite) TestXFSDumpRestic() { + ctx := context.Background() + + // setup a container running the restic rest-server + resticContainer, err := commons.NewTestContainerSetup(ctx, &commons.ResticReq, commons.ResticPort) + xfsTestSuite.Require().NoError(err) + defer func() { + resticErr := resticContainer.Container.Terminate(ctx) + if resticErr != nil { + log.WithError(resticErr).Error("failed to terminate xfs restic container") + } + }() + + xfsConfig := createXFSConfig(resticContainer.Address, resticContainer.Port) + err = viper.ReadConfig(bytes.NewBuffer(xfsConfig)) + xfsTestSuite.Require().NoError(err) + + dirName := fmt.Sprintf("%s/testdir", mountPoint) + + err = os.Mkdir(dirName, 744) + xfsTestSuite.Require().NoError(err) + + err = source.DoBackupForKind(ctx, "xfsdump", false, true, false) + xfsTestSuite.Require().NoError(err) + + err = os.Remove(dirName) + xfsTestSuite.Require().NoError(err) + + err = source.DoRestoreForKind(ctx, "xfsrestore", false, true, false) + xfsTestSuite.Require().NoError(err) + + _, err = os.Stat(dirName) + xfsTestSuite.Require().NoError(err) + err = os.Remove(dumpName) + xfsTestSuite.Require().NoError(err) + err = os.Remove(dirName) + xfsTestSuite.Require().NoError(err) +} + +func TestXFSTestSuite(t *testing.T) { + suite.Run(t, new(XFSTestSuite)) +} + +// createXFSConfig creates a brudi config for the xfs commands +func createXFSConfig(resticIP, resticPort string) []byte { + return []byte(fmt.Sprintf(` +xfsdump: + options: + flags: + level: 0 + destination: %s + additionalArgs: [] + targetFS: %s +xfsrestore: + options: + flags: + source: %s + additionalArgs: [] + destFS: %s +restic: + global: + flags: + repo: rest:http://%s:%s/ + restore: + flags: + target: "/" + id: "latest" +`, dumpName, loopDeviceName, dumpName, mountPoint, resticIP, resticPort)) +}