add more upload session filters (#7933)

* add more upload session filters Signed-off-by: Jörn Friedrich Dreyer <[email protected]> * default descriptions Signed-off-by: Jörn Friedrich Dreyer <[email protected]> * update readme Signed-off-by: Jörn Friedrich Dreyer <[email protected]> * use tablewriter, --json flag Signed-off-by: Jörn Friedrich Dreyer <[email protected]> * update readme Signed-off-by: Jörn Friedrich Dreyer <[email protected]> --------- Signed-off-by: Jörn Friedrich Dreyer <[email protected]>
owncloud · Jan 26, 2024 · 3ed0424 · 3ed0424
1 parent 6cf8546
commit 3ed0424
Show file tree

Hide file tree

Showing 2 changed files with 204 additions and 11 deletions.
diff --git a/services/storage-users/README.md b/services/storage-users/README.md
@@ -36,32 +36,52 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo
 
 Example cases for expired uploads
 
-*   When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared.
+*   In the final step the upload blob is moved from the upload area to the final blobstore (e.g. S3). 
+
 *   If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed.
 
-There are two commands available to manage unfinished uploads
+The admin can restart the postprocessing for this with the postprocessing cli.
+
+The storage users service can only list and clean upload sessions:
 
 ```bash
 ocis storage-users uploads <command>
 ```
 
 ```plaintext
 COMMANDS:
-   list     Print a list of all incomplete uploads
-   clean    Clean up leftovers from expired uploads
+   sessions   Print a list of upload sessions
+   clean      Clean up leftovers from expired uploads
+   list       Print a list of all incomplete uploads (deprecated)
 ```
 
 #### Command Examples
 
-Command to identify incomplete uploads
+Command to list ongoing upload sessions
 
 ```bash
-ocis storage-users uploads list
+ocis storage-users sessions --expired=false
 ```
 
 ```plaintext
-Incomplete uploads:
- - 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00)
+Not expired sessions:
++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+
+|                Space                 |              Upload Id               |  Name   | Offset | Size |              Executant               |                Owner                 |          Expires          | Processing |
++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+
+| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 5e387954-7313-4223-a904-bf996da6ec0b | foo.txt |      0 | 1234 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:04:31+01:00 | false      |
+| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f066244d-97b2-48e7-a30d-b40fcb60cec6 | bar.txt |      0 | 4321 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:18:47+01:00 | false      |
++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+
+```
+
+The sessions command can also output json
+
+```bash
+ocis storage-users sessions --expired=false --json
+```
+
+```json
+{"id":"5e387954-7313-4223-a904-bf996da6ec0b","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"foo.txt","offset":0,"size":1234,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:04:31+01:00","processing":false}
+{"id":"f066244d-97b2-48e7-a30d-b40fcb60cec6","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"bar.txt","offset":0,"size":4321,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:18:47+01:00","processing":false}
 ```
 
 Command to clear expired uploads
@@ -74,6 +94,17 @@ Cleaned uploads:
 - 455bd640-cd08-46e8-a5a0-9304908bd40a (Filename: file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00)
 ```
 
+Deprecated list command to identify unfinished uploads
+
+```bash
+ocis storage-users uploads list
+```
+
+```plaintext
+Incomplete uploads:
+ - 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00)
+```
+
 ### Purge Expired Space Trash-Bins Items
 
 <!-- referencing: https://github.com/owncloud/ocis/pull/5500 -->

diff --git a/services/storage-users/pkg/command/uploads.go b/services/storage-users/pkg/command/uploads.go
@@ -1,12 +1,18 @@
 package command
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"sync"
+	"time"
 
+	tw "github.com/olekukonko/tablewriter"
 	"github.com/urfave/cli/v2"
 
+	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	"github.com/cs3org/reva/v2/pkg/storage"
 	"github.com/cs3org/reva/v2/pkg/storage/fs/registry"
 	"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
@@ -22,6 +28,7 @@ func Uploads(cfg *config.Config) *cli.Command {
 		Usage: "manage unfinished uploads",
 		Subcommands: []*cli.Command{
 			ListUploads(cfg),
+			ListUploadSessions(cfg),
 			PurgeExpiredUploads(cfg),
 		},
 	}
@@ -31,7 +38,7 @@ func Uploads(cfg *config.Config) *cli.Command {
 func ListUploads(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "list",
-		Usage: "Print a list of all incomplete uploads",
+		Usage: "Print a list of all incomplete uploads (deprecated, use sessions)",
 		Before: func(c *cli.Context) error {
 			return configlog.ReturnFatal(parser.ParseConfig(cfg))
 		},
@@ -50,7 +57,7 @@ func ListUploads(cfg *config.Config) *cli.Command {
 
 			managingFS, ok := fs.(storage.UploadSessionLister)
 			if !ok {
-				fmt.Fprintf(os.Stderr, "'%s' storage does not support listing expired uploads\n", cfg.Driver)
+				fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver)
 				os.Exit(1)
 			}
 			expired := false
@@ -69,7 +76,162 @@ func ListUploads(cfg *config.Config) *cli.Command {
 	}
 }
 
-// PurgeExpiredUploads is the entry point for the server command.
+// ListUploadSessions prints a list of upload sessiens
+func ListUploadSessions(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "sessions",
+		Usage: "Print a list of upload sessions",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "id",
+				DefaultText: "unset",
+				Usage:       "filter sessions by upload session id",
+			},
+			&cli.BoolFlag{
+				Name:        "processing",
+				DefaultText: "unset",
+				Usage:       "filter sessions by processing status",
+			},
+			&cli.BoolFlag{
+				Name:        "expired",
+				DefaultText: "unset",
+				Usage:       "filter sessions by expired status",
+			},
+			&cli.BoolFlag{
+				Name:  "json",
+				Usage: "output as json",
+			},
+		},
+		Before: func(c *cli.Context) error {
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
+		},
+		Action: func(c *cli.Context) error {
+			f, ok := registry.NewFuncs[cfg.Driver]
+			if !ok {
+				fmt.Fprintf(os.Stderr, "Unknown filesystem driver '%s'\n", cfg.Driver)
+				os.Exit(1)
+			}
+			drivers := revaconfig.StorageProviderDrivers(cfg)
+			fs, err := f(drivers[cfg.Driver].(map[string]interface{}), nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver)
+				return err
+			}
+
+			managingFS, ok := fs.(storage.UploadSessionLister)
+			if !ok {
+				fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver)
+				os.Exit(1)
+			}
+
+			var b strings.Builder
+			filter := storage.UploadSessionFilter{}
+			if c.IsSet("processing") {
+				processingValue := c.Bool("processing")
+				filter.Processing = &processingValue
+				if !processingValue {
+					b.WriteString("Not ")
+				}
+				if b.Len() == 0 {
+					b.WriteString("Processing ")
+				} else {
+					b.WriteString("processing ")
+				}
+			}
+			if c.IsSet("expired") {
+				expiredValue := c.Bool("expired")
+				filter.Expired = &expiredValue
+				if !expiredValue {
+					if b.Len() == 0 {
+						b.WriteString("Not ")
+					} else {
+						b.WriteString(", not ")
+					}
+				}
+				if b.Len() == 0 {
+					b.WriteString("Expired ")
+				} else {
+					b.WriteString("expired ")
+				}
+			}
+			if b.Len() == 0 {
+				b.WriteString("Sessions")
+			} else {
+				b.WriteString("sessions")
+			}
+			if c.IsSet("id") {
+				idValue := c.String("id")
+				filter.ID = &idValue
+				b.WriteString(" with id '" + idValue + "'")
+			}
+			b.WriteString(":")
+			uploads, err := managingFS.ListUploadSessions(c.Context, filter)
+			if err != nil {
+				return err
+			}
+
+			var table *tw.Table
+			if c.Bool("json") {
+				for _, u := range uploads {
+					ref := u.Reference()
+					s := struct {
+						ID         string         `json:"id"`
+						Space      string         `json:"space"`
+						Filename   string         `json:"filename"`
+						Offset     int64          `json:"offset"`
+						Size       int64          `json:"size"`
+						Executant  userpb.UserId  `json:"executant"`
+						SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"`
+						Expires    time.Time      `json:"expires"`
+						Processing bool           `json:"processing"`
+					}{
+						Space:      ref.GetResourceId().GetSpaceId(),
+						ID:         u.ID(),
+						Filename:   u.Filename(),
+						Offset:     u.Offset(),
+						Size:       u.Size(),
+						Executant:  u.Executant(),
+						SpaceOwner: u.SpaceOwner(),
+						Expires:    u.Expires(),
+						Processing: u.IsProcessing(),
+					}
+					j, err := json.Marshal(s)
+					if err != nil {
+						fmt.Println(err)
+					}
+					fmt.Println(string(j))
+				}
+			} else {
+
+				// Print what the user requested
+				fmt.Println(b.String())
+
+				// start a table
+				table = tw.NewWriter(os.Stdout)
+				table.SetHeader([]string{"Space", "Upload Id", "Name", "Offset", "Size", "Executant", "Owner", "Expires", "Processing"})
+				table.SetAutoFormatHeaders(false)
+
+				for _, u := range uploads {
+					table.Append([]string{
+						u.Reference().ResourceId.GetSpaceId(),
+						u.ID(),
+						u.Filename(),
+						strconv.FormatInt(u.Offset(), 10),
+						strconv.FormatInt(u.Size(), 10),
+						u.Executant().OpaqueId,
+						u.SpaceOwner().GetOpaqueId(),
+						u.Expires().Format(time.RFC3339),
+						strconv.FormatBool(u.IsProcessing()),
+					})
+				}
+				table.Render()
+			}
+			return nil
+		},
+	}
+}
+
+// PurgeExpiredUploads is the entry point for the clean command
 func PurgeExpiredUploads(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "clean",