From 7fd099fd657af4576ca22140c97a3cf45d6e8d1f Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 23 Jan 2017 08:44:12 -0800 Subject: [PATCH 1/7] organizing imports --- system/admin/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/admin/handlers.go b/system/admin/handlers.go index 3195239d..a28b7893 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -21,11 +21,11 @@ import ( "github.com/ponzu-cms/ponzu/system/api" "github.com/ponzu-cms/ponzu/system/db" "github.com/ponzu-cms/ponzu/system/item" - "github.com/tidwall/gjson" "github.com/gorilla/schema" emailer "github.com/nilslice/email" "github.com/nilslice/jwt" + "github.com/tidwall/gjson" ) func adminHandler(res http.ResponseWriter, req *http.Request) { From 35204659b542175d5e4f352c24374f29974cf5fe Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 23 Jan 2017 08:44:33 -0800 Subject: [PATCH 2/7] adding http basic auth fields for db backup via http --- system/admin/config/config.go | 46 +++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/system/admin/config/config.go b/system/admin/config/config.go index 0d557009..3a21e16f 100644 --- a/system/admin/config/config.go +++ b/system/admin/config/config.go @@ -9,18 +9,27 @@ import ( type Config struct { item.Item - Name string `json:"name"` - Domain string `json:"domain"` - HTTPPort string `json:"http_port"` - HTTPSPort string `json:"https_port"` - AdminEmail string `json:"admin_email"` - ClientSecret string `json:"client_secret"` - Etag string `json:"etag"` - DisableCORS bool `json:"cors_disabled"` - DisableGZIP bool `json:"gzip_disabled"` - CacheInvalidate []string `json:"cache"` + Name string `json:"name"` + Domain string `json:"domain"` + HTTPPort string `json:"http_port"` + HTTPSPort string `json:"https_port"` + AdminEmail string `json:"admin_email"` + ClientSecret string `json:"client_secret"` + Etag string `json:"etag"` + DisableCORS bool `json:"cors_disabled"` + DisableGZIP bool `json:"gzip_disabled"` + CacheInvalidate []string `json:"cache"` + BackupBasicAuthUser string `json:"backup_basic_auth_user"` + BackupBasicAuthPassword string `json:"backup_basic_auth_password"` } +const ( + dbBackupInfo = ` +

Database Backup Credentials:

+

Add a user name and password to download a backup of your data via HTTP.

+ ` +) + // String partially implements item.Identifiable and overrides Item's String() func (c *Config) String() string { return c.Name } @@ -97,6 +106,23 @@ func (c *Config) MarshalEditor() ([]byte, error) { "invalidate": "Invalidate Cache", }), }, + editor.Field{ + View: []byte(dbBackupInfo), + }, + editor.Field{ + View: editor.Input("BackupBasicAuthUser", c, map[string]string{ + "label": "HTTP Basic Auth User", + "placeholder": "Enter a user name for Basic Auth access", + "type": "text", + }), + }, + editor.Field{ + View: editor.Input("BackupBasicAuthPassword", c, map[string]string{ + "label": "HTTP Basic Auth Password", + "placeholder": "Enter a password for Basic Auth access", + "type": "password", + }), + }, ) if err != nil { return nil, err From 26aa5815c50890b1c151b968ea60fce7f1c23ba5 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 23 Jan 2017 10:08:55 -0800 Subject: [PATCH 3/7] updating timestamp precision --- system/api/analytics/init.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/system/api/analytics/init.go b/system/api/analytics/init.go index 3eccc134..f24425b9 100644 --- a/system/api/analytics/init.go +++ b/system/api/analytics/init.go @@ -43,13 +43,15 @@ const RANGE = 14 func Record(req *http.Request) { external := strings.Contains(req.URL.Path, "/external/") + ts := int64(time.Nanosecond) * time.Now().UnixNano() / int64(time.Millisecond) + r := apiRequest{ URL: req.URL.String(), Method: req.Method, Origin: req.Header.Get("Origin"), Proto: req.Proto, RemoteAddr: req.RemoteAddr, - Timestamp: time.Now().Unix() * 1000, + Timestamp: ts, External: external, } From 964479e5ce756aa48755dc3de35b4d82326f149f Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 23 Jan 2017 11:45:07 -0800 Subject: [PATCH 4/7] adding basic auth middleware, admin/backup route, and backup routines for system, analytics, and uploads --- system/admin/handlers.go | 32 ++++++++++++++++++++++++++++++++ system/admin/server.go | 4 ++++ 2 files changed, 36 insertions(+) diff --git a/system/admin/handlers.go b/system/admin/handlers.go index a28b7893..a99a62cf 100644 --- a/system/admin/handlers.go +++ b/system/admin/handlers.go @@ -19,6 +19,7 @@ import ( "github.com/ponzu-cms/ponzu/system/admin/upload" "github.com/ponzu-cms/ponzu/system/admin/user" "github.com/ponzu-cms/ponzu/system/api" + "github.com/ponzu-cms/ponzu/system/api/analytics" "github.com/ponzu-cms/ponzu/system/db" "github.com/ponzu-cms/ponzu/system/item" @@ -188,6 +189,37 @@ func configHandler(res http.ResponseWriter, req *http.Request) { } +func backupHandler(res http.ResponseWriter, req *http.Request) { + switch req.URL.Query().Get("source") { + case "system": + err := db.Backup(res) + if err != nil { + log.Println("Failed to run backup on system:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + case "analytics": + err := analytics.Backup(res) + if err != nil { + log.Println("Failed to run backup on analytics:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + case "uploads": + err := upload.Backup(res) + if err != nil { + log.Println("Failed to run backup on uploads:", err) + res.WriteHeader(http.StatusInternalServerError) + return + } + + default: + res.WriteHeader(http.StatusBadRequest) + } +} + func configUsersHandler(res http.ResponseWriter, req *http.Request) { switch req.Method { case http.MethodGet: diff --git a/system/admin/server.go b/system/admin/server.go index 11bfe6f0..df00c211 100644 --- a/system/admin/server.go +++ b/system/admin/server.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "github.com/ponzu-cms/ponzu/system" "github.com/ponzu-cms/ponzu/system/admin/user" "github.com/ponzu-cms/ponzu/system/api" "github.com/ponzu-cms/ponzu/system/db" @@ -52,4 +53,7 @@ func Run() { // through the editor will not load within the admin system. uploadsDir := filepath.Join(pwd, "uploads") http.Handle("/api/uploads/", api.Record(api.CORS(db.CacheControl(http.StripPrefix("/api/uploads/", http.FileServer(restrict(http.Dir(uploadsDir)))))))) + + // Database & uploads backup via HTTP route registered with Basic Auth middleware. + http.HandleFunc("/admin/backup", system.BasicAuth(backupHandler)) } From b750e1e9f7fa16126e3e1a26ed43e1a43e78ce04 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Mon, 23 Jan 2017 11:47:17 -0800 Subject: [PATCH 5/7] adding untracked files for recent basic auth and backup features --- system/admin/upload/backup.go | 78 ++++++++++++++++++++++++++++++++++ system/api/analytics/backup.go | 26 ++++++++++++ system/auth.go | 34 +++++++++++++++ system/db/backup.go | 26 ++++++++++++ 4 files changed, 164 insertions(+) create mode 100644 system/admin/upload/backup.go create mode 100644 system/api/analytics/backup.go create mode 100644 system/auth.go create mode 100644 system/db/backup.go diff --git a/system/admin/upload/backup.go b/system/admin/upload/backup.go new file mode 100644 index 00000000..a6660739 --- /dev/null +++ b/system/admin/upload/backup.go @@ -0,0 +1,78 @@ +package upload + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" +) + +// Backup creates an archive of a project's uploads and writes it +// to the response as a download +func Backup(res http.ResponseWriter) error { + ts := time.Now().Unix() + filename := fmt.Sprintf("uploads-%d.bak.tar.gz", ts) + tmp := os.TempDir() + + // create uploads-{stamp}.bak.tar.gz + f, err := os.Create(filepath.Join(tmp, filename)) + if err != nil { + return err + } + defer f.Close() + + // loop through directory and gzip files + // add all to uploads.bak.tar.gz tarball + gz := gzip.NewWriter(f) + tarball := tar.NewWriter(gz) + err = filepath.Walk("uploads", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + h := &tar.Header{ + Name: info.Name(), + Size: info.Size(), + Mode: int64(info.Mode()), + ModTime: info.ModTime(), + } + + err = tarball.WriteHeader(h) + if err != nil { + return err + } + + src, err := os.Open(path) + if err != nil { + return err + } + + _, err = io.Copy(tarball, src) + + return err + }) + + // write data to response + data, err := os.Open(filepath.Join(tmp, filename)) + if err != nil { + return err + } + defer data.Close() + + disposition := `attachment; filename=%s` + info, err := data.Stat() + if err != nil { + return err + } + + res.Header().Set("Content-Type", "application/octet-stream") + res.Header().Set("Content-Disposition", fmt.Sprintf(disposition, ts)) + res.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size())) + + _, err = io.Copy(res, data) + return err +} diff --git a/system/api/analytics/backup.go b/system/api/analytics/backup.go new file mode 100644 index 00000000..07b1a46a --- /dev/null +++ b/system/api/analytics/backup.go @@ -0,0 +1,26 @@ +package analytics + +import ( + "fmt" + "net/http" + "time" + + "github.com/boltdb/bolt" +) + +// Backup writes a snapshot of the analytics.db database to an HTTP response +func Backup(res http.ResponseWriter) error { + err := store.View(func(tx *bolt.Tx) error { + ts := time.Now().Unix() + disposition := `attachment; filename="analytics-%d.db.bak"` + + res.Header().Set("Content-Type", "application/octet-stream") + res.Header().Set("Content-Disposition", fmt.Sprintf(disposition, ts)) + res.Header().Set("Content-Length", fmt.Sprintf("%d", int(tx.Size()))) + + _, err := tx.WriteTo(res) + return err + }) + + return err +} diff --git a/system/auth.go b/system/auth.go new file mode 100644 index 00000000..cf1adf25 --- /dev/null +++ b/system/auth.go @@ -0,0 +1,34 @@ +package system + +import ( + "net/http" + + "github.com/ponzu-cms/ponzu/system/db" +) + +// BasicAuth adds HTTP Basic Auth check for requests that should implement it +func BasicAuth(next http.HandlerFunc) http.HandlerFunc { + return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + u := db.ConfigCache("backup_basic_auth_user").(string) + p := db.ConfigCache("backup_basic_auth_password").(string) + + if u == "" || p == "" { + res.WriteHeader(http.StatusForbidden) + return + } + + user, password, ok := req.BasicAuth() + + if !ok { + res.WriteHeader(http.StatusForbidden) + return + } + + if u != user || p != password { + res.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(res, req) + }) +} diff --git a/system/db/backup.go b/system/db/backup.go new file mode 100644 index 00000000..735abe4a --- /dev/null +++ b/system/db/backup.go @@ -0,0 +1,26 @@ +package db + +import ( + "fmt" + "net/http" + "time" + + "github.com/boltdb/bolt" +) + +// Backup writes a snapshot of the system.db database to an HTTP response +func Backup(res http.ResponseWriter) error { + err := store.View(func(tx *bolt.Tx) error { + ts := time.Now().Unix() + disposition := `attachment; filename="system-%d.db.bak"` + + res.Header().Set("Content-Type", "application/octet-stream") + res.Header().Set("Content-Disposition", fmt.Sprintf(disposition, ts)) + res.Header().Set("Content-Length", fmt.Sprintf("%d", int(tx.Size()))) + + _, err := tx.WriteTo(res) + return err + }) + + return err +} From e32aca799bd2ef040d70c9af48c222b98d95acdb Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 24 Jan 2017 10:22:32 -0800 Subject: [PATCH 6/7] adding backup support for analytics.db, system.db, and uploads dir --- system/admin/upload/backup.go | 63 +++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/system/admin/upload/backup.go b/system/admin/upload/backup.go index a6660739..28b1b8e2 100644 --- a/system/admin/upload/backup.go +++ b/system/admin/upload/backup.go @@ -17,51 +17,85 @@ func Backup(res http.ResponseWriter) error { ts := time.Now().Unix() filename := fmt.Sprintf("uploads-%d.bak.tar.gz", ts) tmp := os.TempDir() + backup := filepath.Join(tmp, filename) // create uploads-{stamp}.bak.tar.gz - f, err := os.Create(filepath.Join(tmp, filename)) + f, err := os.Create(backup) if err != nil { return err } - defer f.Close() // loop through directory and gzip files // add all to uploads.bak.tar.gz tarball gz := gzip.NewWriter(f) tarball := tar.NewWriter(gz) + err = filepath.Walk("uploads", func(path string, info os.FileInfo, err error) error { if err != nil { return err } - h := &tar.Header{ - Name: info.Name(), - Size: info.Size(), - Mode: int64(info.Mode()), - ModTime: info.ModTime(), - } - - err = tarball.WriteHeader(h) + hdr, err := tar.FileInfoHeader(info, "") if err != nil { return err } - src, err := os.Open(path) + hdr.Name = path + + err = tarball.WriteHeader(hdr) if err != nil { return err } - _, err = io.Copy(tarball, src) + if !info.IsDir() { + src, err := os.Open(path) + if err != nil { + return err + } + defer src.Close() + _, err = io.Copy(tarball, src) + if err != nil { + return err + } - return err + err = tarball.Flush() + if err != nil { + return err + } + + err = gz.Flush() + if err != nil { + return err + } + } + + return nil }) + if err != nil { + fmt.Println(err) + return err + } + + err = gz.Close() + if err != nil { + return err + } + err = tarball.Close() + if err != nil { + return err + } + err = f.Close() + if err != nil { + return err + } // write data to response - data, err := os.Open(filepath.Join(tmp, filename)) + data, err := os.Open(backup) if err != nil { return err } defer data.Close() + defer os.Remove(backup) disposition := `attachment; filename=%s` info, err := data.Stat() @@ -74,5 +108,6 @@ func Backup(res http.ResponseWriter) error { res.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size())) _, err = io.Copy(res, data) + return err } From cc052d38faffd61d36859b84f70437fd3d7b7691 Mon Sep 17 00:00:00 2001 From: Steve Manuel Date: Tue, 24 Jan 2017 10:36:24 -0800 Subject: [PATCH 7/7] bumping version: 0.8.0 --- cmd/ponzu/ponzu.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ponzu/ponzu.json b/cmd/ponzu/ponzu.json index 0f0b1ab5..a997bfb0 100644 --- a/cmd/ponzu/ponzu.json +++ b/cmd/ponzu/ponzu.json @@ -1,3 +1,3 @@ { - "version": "0.7.2" + "version": "0.8.0" } \ No newline at end of file