From 411c90f67e5b9b5acbf76768f05a1ae4b75a6884 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sun, 29 Nov 2020 21:24:36 +0100 Subject: [PATCH] Allow to download files to a temp dir first (#149) Co-authored-by: CrazyMax --- docs/config/download.md | 14 ++++ docs/config/index.md | 3 + internal/config/config_test.go | 5 ++ internal/config/download.go | 2 + internal/config/fixtures/config.test.yml | 1 + internal/grabber/grabber.go | 92 +++++++++++++++--------- internal/grabber/move.go | 11 +++ internal/grabber/move_windows.go | 15 ++++ internal/grabber/perm.go | 25 +++++++ internal/grabber/perm_windows.go | 5 ++ 10 files changed, 139 insertions(+), 34 deletions(-) create mode 100644 internal/grabber/move.go create mode 100644 internal/grabber/move_windows.go create mode 100644 internal/grabber/perm.go create mode 100644 internal/grabber/perm_windows.go diff --git a/docs/config/download.md b/docs/config/download.md index 7d6eb40b..64060a5a 100644 --- a/docs/config/download.md +++ b/docs/config/download.md @@ -15,6 +15,7 @@ since: 2019-02-01T18:50:05Z retry: 3 hideSkipped: false + tempFirst: false createBaseDir: false ``` @@ -150,6 +151,19 @@ Not display skipped downloads. (default: `false`) !!! abstract "Environment variables" * `FTPGRAB_DOWNLOAD_HIDESKIPPED` +## `tempFirst` + +First download the files to a temporary location and then move them to the final destination. (default `false`) + +!!! example "Config file" + ```yaml + download: + tempFirst: false + ``` + +!!! abstract "Environment variables" + * `FTPGRAB_DOWNLOAD_TEMPFIRST` + ## `createBaseDir` Create basename of a FTP source path in the destination folder. This is highly recommended if you have multiple FTP diff --git a/docs/config/index.md b/docs/config/index.md index 40346c41..e4505821 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -57,6 +57,7 @@ You can override this using the [`--config` flag or `CONFIG` env var](../usage/c since: 2019-02-01T18:50:05Z retry: 3 hideSkipped: false + tempFirst: false createBaseDir: false notif: @@ -114,6 +115,7 @@ All configuration from file can be transposed into environment variables. As an since: 2019-02-01T18:50:05Z retry: 3 hideSkipped: false + tempFirst: false createBaseDir: false notif: @@ -154,6 +156,7 @@ Can be transposed to: FTPGRAB_DOWNLOAD_SINCE=2019-02-01T18:50:05Z FTPGRAB_DOWNLOAD_RETRY=3 FTPGRAB_DOWNLOAD_HIDESKIPPED=false + FTPGRAB_DOWNLOAD_TEMPFIRST=false FTPGRAB_DOWNLOAD_CREATEBASEDIR=false FTPGRAB_NOTIF_MAIL_HOST=smtp.example.com diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b12bbbef..82432bd7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -68,6 +68,7 @@ func TestLoadFile(t *testing.T) { SinceTime: time.Date(2019, 2, 1, 18, 50, 05, 0, time.UTC), Retry: 3, HideSkipped: utl.NewFalse(), + TempFirst: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, Notif: &Notif{ @@ -169,6 +170,7 @@ func TestLoadEnv(t *testing.T) { ChmodDir: 0755, Retry: 3, HideSkipped: utl.NewFalse(), + TempFirst: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, }, @@ -206,6 +208,7 @@ func TestLoadEnv(t *testing.T) { ChmodDir: 0755, Retry: 3, HideSkipped: utl.NewFalse(), + TempFirst: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, }, @@ -322,6 +325,7 @@ func TestLoadMixed(t *testing.T) { ChmodDir: 0755, Retry: 3, HideSkipped: utl.NewFalse(), + TempFirst: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, Notif: &Notif{ @@ -373,6 +377,7 @@ func TestLoadMixed(t *testing.T) { ChmodDir: 0755, Retry: 3, HideSkipped: utl.NewTrue(), + TempFirst: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, Notif: &Notif{ diff --git a/internal/config/download.go b/internal/config/download.go index 65f5fd9c..17efcea6 100644 --- a/internal/config/download.go +++ b/internal/config/download.go @@ -20,6 +20,7 @@ type Download struct { SinceTime time.Time `yaml:"-" json:"-" label:"-" file:"-"` Retry int `yaml:"retry,omitempty" json:"retry,omitempty"` HideSkipped *bool `yaml:"hideSkipped,omitempty" json:"hideSkipped,omitempty"` + TempFirst *bool `yaml:"tempFirst,omitempty" json:"tempFirst,omitempty"` CreateBaseDir *bool `yaml:"createBaseDir,omitempty" json:"createBaseDir,omitempty"` } @@ -38,5 +39,6 @@ func (s *Download) SetDefaults() { s.ChmodDir = 0755 s.Retry = 3 s.HideSkipped = utl.NewFalse() + s.TempFirst = utl.NewFalse() s.CreateBaseDir = utl.NewFalse() } diff --git a/internal/config/fixtures/config.test.yml b/internal/config/fixtures/config.test.yml index 615619f6..cbde2d64 100644 --- a/internal/config/fixtures/config.test.yml +++ b/internal/config/fixtures/config.test.yml @@ -20,6 +20,7 @@ download: since: 2019-02-01T18:50:05Z retry: 3 hideSkipped: false + tempFirst: false createBaseDir: false notif: diff --git a/internal/grabber/grabber.go b/internal/grabber/grabber.go index 45821d04..0664ea04 100644 --- a/internal/grabber/grabber.go +++ b/internal/grabber/grabber.go @@ -2,9 +2,9 @@ package grabber import ( "fmt" + "io/ioutil" "os" "path" - "runtime" "time" "github.com/crazy-max/ftpgrab/v7/internal/config" @@ -21,9 +21,10 @@ import ( // Client represents an active grabber object type Client struct { - config *config.Download - db *db.Client - server *server.Client + config *config.Download + db *db.Client + server *server.Client + tempdir string } // New creates new grabber instance @@ -49,10 +50,17 @@ func New(dlConfig *config.Download, dbConfig *config.Db, serverConfig *config.Se return nil, errors.Wrap(err, "Cannot connect to server") } + // Temp dir to download files + tempdir, err := ioutil.TempDir("", ".ftpgrab.*") + if err != nil { + return nil, errors.Wrap(err, "Cannot create temp dir") + } + return &Client{ - config: dlConfig, - db: dbCli, - server: serverCli, + config: dlConfig, + db: dbCli, + server: serverCli, + tempdir: tempdir, }, nil } @@ -115,13 +123,14 @@ func (c *Client) download(file File, retry int) *journal.Entry { sublogger.Warn().Err(err).Msg("Cannot fix parent folder permissions") } - destfile, err := os.Create(destpath) + destfile, err := c.createFile(destpath) if err != nil { sublogger.Error().Err(err).Msg("Cannot create destination file") entry.Level = journal.EntryLevelError entry.Text = fmt.Sprintf("Cannot create destination file: %v", err) return entry } + defer destfile.Close() err = c.server.Retrieve(srcpath, destfile) if err != nil { @@ -135,9 +144,31 @@ func (c *Client) download(file File, retry int) *journal.Entry { return c.download(file, retry) } } else { + if err = destfile.Close(); err != nil { + sublogger.Error().Err(err).Msg("Cannot close destination file") + entry.Level = journal.EntryLevelError + entry.Text = fmt.Sprintf("Cannot close destination file: %v", err) + return entry + } + + if *c.config.TempFirst { + log.Debug(). + Str("tempfile", destfile.Name()). + Str("destfile", destpath). + Msgf("Move temp file") + err := moveFile(destfile.Name(), destpath) + if err != nil { + sublogger.Error().Err(err).Msg("Cannot move file") + entry.Level = journal.EntryLevelError + entry.Text = fmt.Sprintf("Cannot move file: %v", err) + return entry + } + } + sublogger.Info(). Str("duration", time.Since(retrieveStart).Round(time.Millisecond).String()). Msg("File successfully downloaded") + entry.Level = journal.EntryLevelSuccess entry.Text = fmt.Sprintf("%s successfully downloaded in %s", units.HumanSize(float64(file.Info.Size())), @@ -159,6 +190,22 @@ func (c *Client) download(file File, retry int) *journal.Entry { return entry } +func (c *Client) createFile(filename string) (*os.File, error) { + if *c.config.TempFirst { + tempfile, err := ioutil.TempFile(c.tempdir, path.Base(filename)) + if err != nil { + return nil, err + } + return tempfile, nil + } + + destfile, err := os.Create(filename) + if err != nil { + return nil, err + } + return destfile, nil +} + func (c *Client) getStatus(file File) journal.EntryStatus { if !c.isIncluded(file) { return journal.EntryStatusNotIncluded @@ -201,32 +248,6 @@ func (c *Client) isExcluded(file File) bool { return false } -func (c *Client) fixPerms(filepath string) error { - if runtime.GOOS == "windows" { - return nil - } - - fileinfo, err := os.Stat(filepath) - if err != nil { - return err - } - - chmod := os.FileMode(c.config.ChmodFile) - if fileinfo.IsDir() { - chmod = os.FileMode(c.config.ChmodDir) - } - - if err := os.Chmod(filepath, chmod); err != nil { - return err - } - - if err := os.Chown(filepath, c.config.UID, c.config.GID); err != nil { - return err - } - - return nil -} - // Close closes grabber func (c *Client) Close() { if err := c.db.Close(); err != nil { @@ -235,4 +256,7 @@ func (c *Client) Close() { if err := c.server.Close(); err != nil { log.Warn().Err(err).Msg("Cannot close server connection") } + if err := os.RemoveAll(c.tempdir); err != nil { + log.Warn().Err(err).Msg("Cannot remove temp folder") + } } diff --git a/internal/grabber/move.go b/internal/grabber/move.go new file mode 100644 index 00000000..8f520fe0 --- /dev/null +++ b/internal/grabber/move.go @@ -0,0 +1,11 @@ +// +build !windows + +package grabber + +import ( + "os" +) + +func moveFile(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} diff --git a/internal/grabber/move_windows.go b/internal/grabber/move_windows.go new file mode 100644 index 00000000..246fc97e --- /dev/null +++ b/internal/grabber/move_windows.go @@ -0,0 +1,15 @@ +package grabber + +import "syscall" + +func moveFile(oldpath, newpath string) error { + from, err := syscall.UTF16PtrFromString(oldpath) + if err != nil { + return err + } + to, err := syscall.UTF16PtrFromString(newpath) + if err != nil { + return err + } + return syscall.MoveFile(from, to) +} diff --git a/internal/grabber/perm.go b/internal/grabber/perm.go new file mode 100644 index 00000000..dcf8d941 --- /dev/null +++ b/internal/grabber/perm.go @@ -0,0 +1,25 @@ +// +build !windows + +package grabber + +import ( + "os" +) + +func (c *Client) fixPerms(filepath string) error { + fileinfo, err := os.Stat(filepath) + if err != nil { + return err + } + + chmod := os.FileMode(c.config.ChmodFile) + if fileinfo.IsDir() { + chmod = os.FileMode(c.config.ChmodDir) + } + + if err = os.Chmod(filepath, chmod); err != nil { + return err + } + + return os.Chown(filepath, c.config.UID, c.config.GID) +} diff --git a/internal/grabber/perm_windows.go b/internal/grabber/perm_windows.go new file mode 100644 index 00000000..138d6951 --- /dev/null +++ b/internal/grabber/perm_windows.go @@ -0,0 +1,5 @@ +package grabber + +func (c *Client) fixPerms(filepath string) error { + return nil +}