Allow to download files to a temp dir first (#149)

Co-authored-by: CrazyMax <[email protected]>

docs/config/download.md

@@ -15,6 +15,7 @@
       since: 2019-02-01T18:50:05Z
       retry: 3
       hideSkipped: false
+      tempFirst: false
       createBaseDir: false
     ```
 
@@ -150,6 +151,19 @@ Not display skipped downloads. (default: `false`)
 !!! abstract "Environment variables"
     * `FTPGRAB_DOWNLOAD_HIDESKIPPED`
 
+## `tempFirst`
+
+First download the files to a temporary location and then move them to the final destination. (default `false`)
+
+!!! example "Config file"
+    ```yaml
+    download:
+      tempFirst: false
+    ```
+
+!!! abstract "Environment variables"
+    * `FTPGRAB_DOWNLOAD_TEMPFIRST`
+
 ## `createBaseDir`
 
 Create basename of a FTP source path in the destination folder. This is highly recommended if you have multiple FTP

docs/config/index.md

@@ -57,6 +57,7 @@ You can override this using the [`--config` flag or `CONFIG` env var](../usage/c
       since: 2019-02-01T18:50:05Z
       retry: 3
       hideSkipped: false
+      tempFirst: false
       createBaseDir: false
 
     notif:
@@ -114,6 +115,7 @@ All configuration from file can be transposed into environment variables. As an
       since: 2019-02-01T18:50:05Z
       retry: 3
       hideSkipped: false
+      tempFirst: false
       createBaseDir: false
 
     notif:
@@ -154,6 +156,7 @@ Can be transposed to:
     FTPGRAB_DOWNLOAD_SINCE=2019-02-01T18:50:05Z
     FTPGRAB_DOWNLOAD_RETRY=3
     FTPGRAB_DOWNLOAD_HIDESKIPPED=false
+    FTPGRAB_DOWNLOAD_TEMPFIRST=false
     FTPGRAB_DOWNLOAD_CREATEBASEDIR=false
 
     FTPGRAB_NOTIF_MAIL_HOST=smtp.example.com

internal/config/config_test.go

@@ -68,6 +68,7 @@ func TestLoadFile(t *testing.T) {
 					SinceTime:     time.Date(2019, 2, 1, 18, 50, 05, 0, time.UTC),
 					Retry:         3,
 					HideSkipped:   utl.NewFalse(),
+					TempFirst:     utl.NewFalse(),
 					CreateBaseDir: utl.NewFalse(),
 				},
 				Notif: &Notif{
@@ -169,6 +170,7 @@ func TestLoadEnv(t *testing.T) {
 					ChmodDir:      0755,
 					Retry:         3,
 					HideSkipped:   utl.NewFalse(),
+					TempFirst:     utl.NewFalse(),
 					CreateBaseDir: utl.NewFalse(),
 				},
 			},
@@ -206,6 +208,7 @@ func TestLoadEnv(t *testing.T) {
 					ChmodDir:      0755,
 					Retry:         3,
 					HideSkipped:   utl.NewFalse(),
+					TempFirst:     utl.NewFalse(),
 					CreateBaseDir: utl.NewFalse(),
 				},
 			},
@@ -322,6 +325,7 @@ func TestLoadMixed(t *testing.T) {
 					ChmodDir:      0755,
 					Retry:         3,
 					HideSkipped:   utl.NewFalse(),
+					TempFirst:     utl.NewFalse(),
 					CreateBaseDir: utl.NewFalse(),
 				},
 				Notif: &Notif{
@@ -373,6 +377,7 @@ func TestLoadMixed(t *testing.T) {
 					ChmodDir:      0755,
 					Retry:         3,
 					HideSkipped:   utl.NewTrue(),
+					TempFirst:     utl.NewFalse(),
 					CreateBaseDir: utl.NewFalse(),
 				},
 				Notif: &Notif{

internal/config/download.go

@@ -20,6 +20,7 @@ type Download struct {
 	SinceTime     time.Time `yaml:"-" json:"-" label:"-" file:"-"`
 	Retry         int       `yaml:"retry,omitempty" json:"retry,omitempty"`
 	HideSkipped   *bool     `yaml:"hideSkipped,omitempty" json:"hideSkipped,omitempty"`
+	TempFirst     *bool     `yaml:"tempFirst,omitempty" json:"tempFirst,omitempty"`
 	CreateBaseDir *bool     `yaml:"createBaseDir,omitempty" json:"createBaseDir,omitempty"`
 }
 
@@ -38,5 +39,6 @@ func (s *Download) SetDefaults() {
 	s.ChmodDir = 0755
 	s.Retry = 3
 	s.HideSkipped = utl.NewFalse()
+	s.TempFirst = utl.NewFalse()
 	s.CreateBaseDir = utl.NewFalse()
 }

internal/config/fixtures/config.test.yml

@@ -20,6 +20,7 @@ download:
   since: 2019-02-01T18:50:05Z
   retry: 3
   hideSkipped: false
+  tempFirst: false
   createBaseDir: false
 
 notif:

internal/grabber/grabber.go

@@ -2,9 +2,9 @@ package grabber
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
-	"runtime"
 	"time"
 
 	"github.com/crazy-max/ftpgrab/v7/internal/config"
@@ -21,9 +21,10 @@ import (
 
 // Client represents an active grabber object
 type Client struct {
-	config *config.Download
-	db     *db.Client
-	server *server.Client
+	config  *config.Download
+	db      *db.Client
+	server  *server.Client
+	tempdir string
 }
 
 // New creates new grabber instance
@@ -49,10 +50,17 @@ func New(dlConfig *config.Download, dbConfig *config.Db, serverConfig *config.Se
 		return nil, errors.Wrap(err, "Cannot connect to server")
 	}
 
+	// Temp dir to download files
+	tempdir, err := ioutil.TempDir("", ".ftpgrab.*")
+	if err != nil {
+		return nil, errors.Wrap(err, "Cannot create temp dir")
+	}
+
 	return &Client{
-		config: dlConfig,
-		db:     dbCli,
-		server: serverCli,
+		config:  dlConfig,
+		db:      dbCli,
+		server:  serverCli,
+		tempdir: tempdir,
 	}, nil
 }
 
@@ -115,13 +123,14 @@ func (c *Client) download(file File, retry int) *journal.Entry {
 		sublogger.Warn().Err(err).Msg("Cannot fix parent folder permissions")
 	}
 
-	destfile, err := os.Create(destpath)
+	destfile, err := c.createFile(destpath)
 	if err != nil {
 		sublogger.Error().Err(err).Msg("Cannot create destination file")
 		entry.Level = journal.EntryLevelError
 		entry.Text = fmt.Sprintf("Cannot create destination file: %v", err)
 		return entry
 	}
+	defer destfile.Close()
 
 	err = c.server.Retrieve(srcpath, destfile)
 	if err != nil {
@@ -135,9 +144,31 @@ func (c *Client) download(file File, retry int) *journal.Entry {
 			return c.download(file, retry)
 		}
 	} else {
+		if err = destfile.Close(); err != nil {
+			sublogger.Error().Err(err).Msg("Cannot close destination file")
+			entry.Level = journal.EntryLevelError
+			entry.Text = fmt.Sprintf("Cannot close destination file: %v", err)
+			return entry
+		}
+
+		if *c.config.TempFirst {
+			log.Debug().
+				Str("tempfile", destfile.Name()).
+				Str("destfile", destpath).
+				Msgf("Move temp file")
+			err := moveFile(destfile.Name(), destpath)
+			if err != nil {
+				sublogger.Error().Err(err).Msg("Cannot move file")
+				entry.Level = journal.EntryLevelError
+				entry.Text = fmt.Sprintf("Cannot move file: %v", err)
+				return entry
+			}
+		}
+
 		sublogger.Info().
 			Str("duration", time.Since(retrieveStart).Round(time.Millisecond).String()).
 			Msg("File successfully downloaded")
+
 		entry.Level = journal.EntryLevelSuccess
 		entry.Text = fmt.Sprintf("%s successfully downloaded in %s",
 			units.HumanSize(float64(file.Info.Size())),
@@ -159,6 +190,22 @@ func (c *Client) download(file File, retry int) *journal.Entry {
 	return entry
 }
 
+func (c *Client) createFile(filename string) (*os.File, error) {
+	if *c.config.TempFirst {
+		tempfile, err := ioutil.TempFile(c.tempdir, path.Base(filename))
+		if err != nil {
+			return nil, err
+		}
+		return tempfile, nil
+	}
+
+	destfile, err := os.Create(filename)
+	if err != nil {
+		return nil, err
+	}
+	return destfile, nil
+}
+
 func (c *Client) getStatus(file File) journal.EntryStatus {
 	if !c.isIncluded(file) {
 		return journal.EntryStatusNotIncluded
@@ -201,32 +248,6 @@ func (c *Client) isExcluded(file File) bool {
 	return false
 }
 
-func (c *Client) fixPerms(filepath string) error {
-	if runtime.GOOS == "windows" {
-		return nil
-	}
-
-	fileinfo, err := os.Stat(filepath)
-	if err != nil {
-		return err
-	}
-
-	chmod := os.FileMode(c.config.ChmodFile)
-	if fileinfo.IsDir() {
-		chmod = os.FileMode(c.config.ChmodDir)
-	}
-
-	if err := os.Chmod(filepath, chmod); err != nil {
-		return err
-	}
-
-	if err := os.Chown(filepath, c.config.UID, c.config.GID); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // Close closes grabber
 func (c *Client) Close() {
 	if err := c.db.Close(); err != nil {
@@ -235,4 +256,7 @@ func (c *Client) Close() {
 	if err := c.server.Close(); err != nil {
 		log.Warn().Err(err).Msg("Cannot close server connection")
 	}
+	if err := os.RemoveAll(c.tempdir); err != nil {
+		log.Warn().Err(err).Msg("Cannot remove temp folder")
+	}
 }

internal/grabber/move.go

@@ -0,0 +1,11 @@
+// +build !windows
+
+package grabber
+
+import (
+	"os"
+)
+
+func moveFile(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}

internal/grabber/move_windows.go

@@ -0,0 +1,15 @@
+package grabber
+
+import "syscall"
+
+func moveFile(oldpath, newpath string) error {
+	from, err := syscall.UTF16PtrFromString(oldpath)
+	if err != nil {
+		return err
+	}
+	to, err := syscall.UTF16PtrFromString(newpath)
+	if err != nil {
+		return err
+	}
+	return syscall.MoveFile(from, to)
+}

internal/grabber/perm.go

@@ -0,0 +1,25 @@
+// +build !windows
+
+package grabber
+
+import (
+	"os"
+)
+
+func (c *Client) fixPerms(filepath string) error {
+	fileinfo, err := os.Stat(filepath)
+	if err != nil {
+		return err
+	}
+
+	chmod := os.FileMode(c.config.ChmodFile)
+	if fileinfo.IsDir() {
+		chmod = os.FileMode(c.config.ChmodDir)
+	}
+
+	if err = os.Chmod(filepath, chmod); err != nil {
+		return err
+	}
+
+	return os.Chown(filepath, c.config.UID, c.config.GID)
+}

internal/grabber/perm_windows.go

@@ -0,0 +1,5 @@
+package grabber
+
+func (c *Client) fixPerms(filepath string) error {
+	return nil
+}