From 4da5203580cb71312bc589755051afb6dcc0663f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 8 Nov 2024 09:44:19 +0100 Subject: [PATCH 1/8] async propagation polishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decrease noise Signed-off-by: Jörn Friedrich Dreyer use more robust space root check Signed-off-by: Jörn Friedrich Dreyer add changelog Signed-off-by: Jörn Friedrich Dreyer fix typo Signed-off-by: Jörn Friedrich Dreyer revert import reordering Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/propagation-polishing.md | 6 ++++++ pkg/storage/utils/decomposedfs/node/node.go | 3 +-- pkg/storage/utils/decomposedfs/tree/propagator/async.go | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/propagation-polishing.md diff --git a/changelog/unreleased/propagation-polishing.md b/changelog/unreleased/propagation-polishing.md new file mode 100644 index 0000000000..c5efcedef2 --- /dev/null +++ b/changelog/unreleased/propagation-polishing.md @@ -0,0 +1,6 @@ +Enhancement: polish propagation related code + +We polished some corner cases for propagation that reduce log messages in normal operation. + +https://github.com/cs3org/reva/pull/4921 + diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 795ce65c0e..0cf113bc8d 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -1261,8 +1261,7 @@ func (n *Node) ProcessingID(ctx context.Context) (string, error) { // IsSpaceRoot checks if the node is a space root func (n *Node) IsSpaceRoot(ctx context.Context) bool { - _, err := n.Xattr(ctx, prefixes.SpaceNameAttr) - return err == nil + return n.ID == n.SpaceID } // SetScanData sets the virus scan info to the node diff --git a/pkg/storage/utils/decomposedfs/tree/propagator/async.go b/pkg/storage/utils/decomposedfs/tree/propagator/async.go index 2f55b1b99a..14e65dea06 100644 --- a/pkg/storage/utils/decomposedfs/tree/propagator/async.go +++ b/pkg/storage/utils/decomposedfs/tree/propagator/async.go @@ -183,7 +183,7 @@ func (p AsyncPropagator) queuePropagation(ctx context.Context, spaceID, nodeID s ready = true break } - log.Error().Err(err).Msg("failed to write Change to disk (retrying)") + log.Debug().Err(err).Msg("failed to write Change to disk (retrying)") err = os.Mkdir(filepath.Dir(changePath), 0700) triggerPropagation = err == nil || os.IsExist(err) // only the first goroutine, which succeeds to create the directory, is supposed to actually trigger the propagation } @@ -386,7 +386,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, spaceID, nodeID string, // a negative new treesize. Something must have gone wrong with the accounting. // Reset the current treesize to 0. log.Error().Uint64("treeSize", treeSize).Int64("sizeDiff", pc.SizeDiff). - Msg("Error when updating treesize of node. Updated treesize < 0. Reestting to 0") + Msg("Error when updating treesize of node. Updated treesize < 0. Resetting to 0") newSize = 0 default: newSize = treeSize - uint64(-pc.SizeDiff) @@ -414,7 +414,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, spaceID, nodeID string, log.Info().Msg("Propagation done. cleaning up") cleanup() - if !n.IsSpaceRoot(ctx) { // This does not seem robust as it checks the space name property + if !n.IsSpaceRoot(ctx) { p.queuePropagation(ctx, n.SpaceID, n.ParentID, pc, log) } From 6f8f60371f291aa1c1b3b3520b5cecef151f91fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 20 Nov 2024 08:58:28 +0100 Subject: [PATCH 2/8] Handle permission denied errors when touching the new file Co-authored-by: Christian Richter --- internal/http/services/appprovider/appprovider.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 9d93b18621..1ad71290cf 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -237,6 +237,10 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { } if touchRes.Status.Code != rpc.Code_CODE_OK { + if touchRes.Status.Code == rpc.Code_CODE_PERMISSION_DENIED { + writeError(w, r, appErrorPermissionDenied, "touching the file failed", nil) + return + } writeError(w, r, appErrorServerError, "touching the file failed", nil) return } From cc41e7badff3b48ba9c30579d75f9395f6dc5b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 20 Nov 2024 10:38:23 +0100 Subject: [PATCH 3/8] Add changelog --- changelog/unreleased/fix-approvider-error.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/fix-approvider-error.md diff --git a/changelog/unreleased/fix-approvider-error.md b/changelog/unreleased/fix-approvider-error.md new file mode 100644 index 0000000000..ec65f93ec5 --- /dev/null +++ b/changelog/unreleased/fix-approvider-error.md @@ -0,0 +1,5 @@ +Bugfix: Fix a wrong error code when approvider creates a new file + +We fixed a problem where the approvider would return a 500 error instead of 403 when trying to create a new file in a read-only share. + +https://github.com/cs3org/reva/pull/4964 From 336e87125c68fbfbd9ba840cb386dc4f32771536 Mon Sep 17 00:00:00 2001 From: Christian Richter <1058116+dragonchaser@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:59:06 +0100 Subject: [PATCH 4/8] Update internal/http/services/appprovider/appprovider.go --- internal/http/services/appprovider/appprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 1ad71290cf..c434eb6cdc 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -238,7 +238,7 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { if touchRes.Status.Code != rpc.Code_CODE_OK { if touchRes.Status.Code == rpc.Code_CODE_PERMISSION_DENIED { - writeError(w, r, appErrorPermissionDenied, "touching the file failed", nil) + writeError(w, r, appErrorPermissionDenied, "permission denied to create the file", nil) return } writeError(w, r, appErrorServerError, "touching the file failed", nil) From 30600cf5c219c4c8725ef7fbdb343f51c849a581 Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Wed, 20 Nov 2024 13:14:34 +0100 Subject: [PATCH 5/8] release-2.26.7 --- CHANGELOG.md | 106 +++++++++++------- RELEASE_DATE | 2 +- VERSION | 2 +- .../fix-approvider-error.md | 0 changelog/NOTE.md | 85 ++------------ .../en/docs/changelog/2.26.7/_index.md | 30 +++++ .../grpc/services/storageprovider/_index.md | 106 +++++++++--------- 7 files changed, 156 insertions(+), 175 deletions(-) rename changelog/{unreleased => 2.26.7_2024-11-20}/fix-approvider-error.md (100%) create mode 100644 docs/content/en/docs/changelog/2.26.7/_index.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fa5eae2522..ab703dcdbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +Changelog for reva 2.26.7 (2024-11-20) +======================================= + +The following sections list the changes in reva 2.26.7 relevant to +reva users. The changes are ordered by importance. + +Summary +------- + +* Fix #4964: Fix a wrong error code when approvider creates a new file + +Details +------- + +* Bugfix #4964: Fix a wrong error code when approvider creates a new file + + We fixed a problem where the approvider would return a 500 error instead of 403 when trying to + create a new file in a read-only share. + + https://github.com/cs3org/reva/pull/4964 + Changelog for reva 2.26.6 (2024-11-19) ======================================= @@ -12,11 +33,11 @@ Summary * Fix #4953: Avoid gateway panics * Fix #4959: Fix missing file touched event * Fix #4933: Fix federated sharing when using an external identity provider +* Fix #4956: Improve posixfs error handling and logging * Fix #4935: Enable datatx log * Fix #4936: Do not delete mlock files -* Fix #4954: Prevent a panic when logging an error -* Fix #4956: Improve posixfs error handling and logging * Fix #4951: Pass the initialized logger down the stack +* Fix #4954: Prevent a panic when logging an error Details ------- @@ -57,6 +78,12 @@ Details https://github.com/cs3org/reva/pull/4933 +* Bugfix #4956: Improve posixfs error handling and logging + + We improved error handling and logging in the posixfs storage driver. + + https://github.com/cs3org/reva/pull/4956 + * Bugfix #4935: Enable datatx log We now pass a properly initialized logger to the datatx implementations, allowing the tus @@ -72,18 +99,6 @@ Details https://github.com/cs3org/reva/pull/4936 https://github.com/cs3org/reva/pull/4924 -* Bugfix #4954: Prevent a panic when logging an error - - We fixed a panic when constructing a path failed to get the parent for a node. - - https://github.com/cs3org/reva/pull/4954 - -* Bugfix #4956: Improve posixfs error handling and logging - - We improved error handling and logging in the posixfs storage driver. - - https://github.com/cs3org/reva/pull/4956 - * Bugfix #4951: Pass the initialized logger down the stack We now make the initialized logger available to grpc services and storage drivers, which @@ -91,6 +106,12 @@ Details https://github.com/cs3org/reva/pull/4951 +* Bugfix #4954: Prevent a panic when logging an error + + We fixed a panic when constructing a path failed to get the parent for a node. + + https://github.com/cs3org/reva/pull/4954 + Changelog for reva 2.26.5 (2024-11-12) ======================================= @@ -5159,6 +5180,34 @@ Details https://github.com/cs3org/reva/pull/3083 +Changelog for reva 2.7.1 (2022-07-15) +======================================= + +The following sections list the changes in reva 2.7.1 relevant to +reva users. The changes are ordered by importance. + +Summary +------- + +* Fix #3080: Make dataproviders return more headers +* Enh #3046: Add user filter + +Details +------- + +* Bugfix #3080: Make dataproviders return more headers + + Instead of ocdav doing an additional Stat request we now rely on the dataprovider to return the + necessary metadata information as headers. + + https://github.com/owncloud/reva/issues/3080 + +* Enhancement #3046: Add user filter + + This PR adds the ability to filter spaces by user-id + + https://github.com/cs3org/reva/pull/3046 + Changelog for reva 2.7.0 (2022-07-15) ======================================= @@ -5291,34 +5340,6 @@ Details https://github.com/owncloud/ocis/issues/3073 https://github.com/cs3org/reva/pull/2977 -Changelog for reva 2.7.1 (2022-07-15) -======================================= - -The following sections list the changes in reva 2.7.1 relevant to -reva users. The changes are ordered by importance. - -Summary -------- - -* Fix #3080: Make dataproviders return more headers -* Enh #3046: Add user filter - -Details -------- - -* Bugfix #3080: Make dataproviders return more headers - - Instead of ocdav doing an additional Stat request we now rely on the dataprovider to return the - necessary metadata information as headers. - - https://github.com/owncloud/reva/issues/3080 - -* Enhancement #3046: Add user filter - - This PR adds the ability to filter spaces by user-id - - https://github.com/cs3org/reva/pull/3046 - Changelog for reva 2.6.1 (2022-06-27) ======================================= @@ -11571,3 +11592,4 @@ Details from Drone into Github pages. https://github.com/cs3org/reva/pull/334 + diff --git a/RELEASE_DATE b/RELEASE_DATE index ae87a9fe77..0a5877fa56 100644 --- a/RELEASE_DATE +++ b/RELEASE_DATE @@ -1 +1 @@ -2024-11-19 \ No newline at end of file +2024-11-20 \ No newline at end of file diff --git a/VERSION b/VERSION index 06994f8fbe..95bac8a8bc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.26.6 \ No newline at end of file +2.26.7 \ No newline at end of file diff --git a/changelog/unreleased/fix-approvider-error.md b/changelog/2.26.7_2024-11-20/fix-approvider-error.md similarity index 100% rename from changelog/unreleased/fix-approvider-error.md rename to changelog/2.26.7_2024-11-20/fix-approvider-error.md diff --git a/changelog/NOTE.md b/changelog/NOTE.md index f158652834..9c06b7434b 100644 --- a/changelog/NOTE.md +++ b/changelog/NOTE.md @@ -1,92 +1,21 @@ -Changelog for reva 2.26.6 (2024-11-19) +Changelog for reva 2.26.7 (2024-11-20) ======================================= -The following sections list the changes in reva 2.26.6 relevant to +The following sections list the changes in reva 2.26.7 relevant to reva users. The changes are ordered by importance. Summary ------- -* Fix #4955: Allow small clock skew in reva token validation -* Fix #4929: Fix flaky posixfs integration tests -* Fix #4953: Avoid gateway panics -* Fix #4959: Fix missing file touched event -* Fix #4933: Fix federated sharing when using an external identity provider -* Fix #4935: Enable datatx log -* Fix #4936: Do not delete mlock files -* Fix #4954: Prevent a panic when logging an error -* Fix #4956: Improve posixfs error handling and logging -* Fix #4951: Pass the initialized logger down the stack +* Fix #4964: Fix a wrong error code when approvider creates a new file Details ------- -* Bugfix #4955: Allow small clock skew in reva token validation +* Bugfix #4964: Fix a wrong error code when approvider creates a new file - Allow for a small clock skew (3 seconds by default) when validating reva tokens as the different - services might be running on different machines. + We fixed a problem where the approvider would return a 500 error instead of 403 when trying to + create a new file in a read-only share. - https://github.com/cs3org/reva/issues/4952 - https://github.com/cs3org/reva/pull/4955 + https://github.com/cs3org/reva/pull/4964 -* Bugfix #4929: Fix flaky posixfs integration tests - - We fixed a problem with the posixfs integration tests where the in-memory id cache sometimes - hadn't caught up with the cleanup between test runs leading to flaky failures. - - https://github.com/cs3org/reva/pull/4929 - -* Bugfix #4953: Avoid gateway panics - - The gateway would panic if there is a missing user in the context. Now it errors instead. - - https://github.com/cs3org/reva/issues/4953 - -* Bugfix #4959: Fix missing file touched event - - We have fixed an issue where the `file touched` event was not being triggered when an office - document was created. - - https://github.com/owncloud/ocis/issues/8950 - https://github.com/cs3org/reva/pull/4959 - -* Bugfix #4933: Fix federated sharing when using an external identity provider - - We fixes and issue that caused federated sharing to fail when the identity provider url did not - match the federation provider url. - - https://github.com/cs3org/reva/pull/4933 - -* Bugfix #4935: Enable datatx log - - We now pass a properly initialized logger to the datatx implementations, allowing the tus - handler to log with the same level as the rest of reva. - - https://github.com/cs3org/reva/pull/4935 - -* Bugfix #4936: Do not delete mlock files - - To prevent stale NFS file handles we no longer delete empty mlock files after updating the - metadata. - - https://github.com/cs3org/reva/pull/4936 - https://github.com/cs3org/reva/pull/4924 - -* Bugfix #4954: Prevent a panic when logging an error - - We fixed a panic when constructing a path failed to get the parent for a node. - - https://github.com/cs3org/reva/pull/4954 - -* Bugfix #4956: Improve posixfs error handling and logging - - We improved error handling and logging in the posixfs storage driver. - - https://github.com/cs3org/reva/pull/4956 - -* Bugfix #4951: Pass the initialized logger down the stack - - We now make the initialized logger available to grpc services and storage drivers, which - allows for easier and more uniform logging. - - https://github.com/cs3org/reva/pull/4951 diff --git a/docs/content/en/docs/changelog/2.26.7/_index.md b/docs/content/en/docs/changelog/2.26.7/_index.md new file mode 100644 index 0000000000..75ffd6feea --- /dev/null +++ b/docs/content/en/docs/changelog/2.26.7/_index.md @@ -0,0 +1,30 @@ + +--- +title: "v2.26.7" +linkTitle: "v2.26.7" +weight: 40 +description: > + Changelog for Reva v2.26.7 (2024-11-20) +--- + +Changelog for reva 2.26.7 (2024-11-20) +======================================= + +The following sections list the changes in reva 2.26.7 relevant to +reva users. The changes are ordered by importance. + +Summary +------- + +* Fix #4964: Fix a wrong error code when approvider creates a new file + +Details +------- + +* Bugfix #4964: Fix a wrong error code when approvider creates a new file + + We fixed a problem where the approvider would return a 500 error instead of 403 when trying to + create a new file in a read-only share. + + https://github.com/cs3org/reva/pull/4964 + diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index 1a6eb0ea7f..11dc5e468a 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -6,130 +6,130 @@ description: > Configuration for the storageprovider service --- -# _struct: eventconfig_ +# _struct: config_ -{{% dir name="nats_address" type="string" default="address of the nats server" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L77) +{{% dir name="driver" type="string" default="localhome" %}} +The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) {{< highlight toml >}} [grpc.services.storageprovider] -nats_address = "address of the nats server" +driver = "localhome" {{< /highlight >}} {{% /dir %}} -{{% dir name="nats_clusterid" type="string" default="clusterid of the nats server" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L78) +{{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} -[grpc.services.storageprovider] -nats_clusterid = "clusterid of the nats server" +[grpc.services.storageprovider.drivers.localhome] +root = "/var/tmp/reva/" +share_folder = "/MyShares" +user_layout = "{{.Username}}" + {{< /highlight >}} {{% /dir %}} -{{% dir name="tls_insecure" type="bool" default=Whether to verify the server TLS certificates. %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L79) +{{% dir name="data_server_url" type="string" default="http://localhost/data" %}} +The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L67) {{< highlight toml >}} [grpc.services.storageprovider] -tls_insecure = Whether to verify the server TLS certificates. +data_server_url = "http://localhost/data" {{< /highlight >}} {{% /dir %}} -{{% dir name="tls_root_ca_cert" type="string" default="The root CA certificate used to validate the server's TLS certificate." %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L80) +{{% dir name="expose_data_server" type="bool" default=false %}} +Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L68) {{< highlight toml >}} [grpc.services.storageprovider] -tls_root_ca_cert = "The root CA certificate used to validate the server's TLS certificate." +expose_data_server = false {{< /highlight >}} {{% /dir %}} -{{% dir name="nats_enable_tls" type="bool" default=events tls switch %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L81) +{{% dir name="available_checksums" type="map[string]uint32" default=nil %}} +List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L69) {{< highlight toml >}} [grpc.services.storageprovider] -nats_enable_tls = events tls switch +available_checksums = nil {{< /highlight >}} {{% /dir %}} -{{% dir name="nats_username" type="string" default="event stream username" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L82) +{{% dir name="custom_mimetypes_json" type="string" default="nil" %}} +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L70) {{< highlight toml >}} [grpc.services.storageprovider] -nats_username = "event stream username" +custom_mimetypes_json = "nil" {{< /highlight >}} {{% /dir %}} -{{% dir name="nats_password" type="string" default="event stream password" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L83) +{{% dir name="upload_expiration" type="int64" default=0 %}} +Duration for how long uploads will be valid. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L72) {{< highlight toml >}} [grpc.services.storageprovider] -nats_password = "event stream password" +upload_expiration = 0 {{< /highlight >}} {{% /dir %}} -# _struct: config_ - -{{% dir name="driver" type="string" default="localhome" %}} -The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) +{{% dir name="events" type="eventconfig" default=0 %}} +Event stream configuration [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L73) {{< highlight toml >}} [grpc.services.storageprovider] -driver = "localhome" +events = 0 {{< /highlight >}} {{% /dir %}} -{{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) -{{< highlight toml >}} -[grpc.services.storageprovider.drivers.localhome] -root = "/var/tmp/reva/" -share_folder = "/MyShares" -user_layout = "{{.Username}}" +# _struct: eventconfig_ +{{% dir name="nats_address" type="string" default="address of the nats server" %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L77) +{{< highlight toml >}} +[grpc.services.storageprovider] +nats_address = "address of the nats server" {{< /highlight >}} {{% /dir %}} -{{% dir name="data_server_url" type="string" default="http://localhost/data" %}} -The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L67) +{{% dir name="nats_clusterid" type="string" default="clusterid of the nats server" %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L78) {{< highlight toml >}} [grpc.services.storageprovider] -data_server_url = "http://localhost/data" +nats_clusterid = "clusterid of the nats server" {{< /highlight >}} {{% /dir %}} -{{% dir name="expose_data_server" type="bool" default=false %}} -Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L68) +{{% dir name="tls_insecure" type="bool" default=Whether to verify the server TLS certificates. %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L79) {{< highlight toml >}} [grpc.services.storageprovider] -expose_data_server = false +tls_insecure = Whether to verify the server TLS certificates. {{< /highlight >}} {{% /dir %}} -{{% dir name="available_checksums" type="map[string]uint32" default=nil %}} -List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L69) +{{% dir name="tls_root_ca_cert" type="string" default="The root CA certificate used to validate the server's TLS certificate." %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L80) {{< highlight toml >}} [grpc.services.storageprovider] -available_checksums = nil +tls_root_ca_cert = "The root CA certificate used to validate the server's TLS certificate." {{< /highlight >}} {{% /dir %}} -{{% dir name="custom_mimetypes_json" type="string" default="nil" %}} -An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L70) +{{% dir name="nats_enable_tls" type="bool" default=events tls switch %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L81) {{< highlight toml >}} [grpc.services.storageprovider] -custom_mimetypes_json = "nil" +nats_enable_tls = events tls switch {{< /highlight >}} {{% /dir %}} -{{% dir name="upload_expiration" type="int64" default=0 %}} -Duration for how long uploads will be valid. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L72) +{{% dir name="nats_username" type="string" default="event stream username" %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L82) {{< highlight toml >}} [grpc.services.storageprovider] -upload_expiration = 0 +nats_username = "event stream username" {{< /highlight >}} {{% /dir %}} -{{% dir name="events" type="eventconfig" default=0 %}} -Event stream configuration [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L73) +{{% dir name="nats_password" type="string" default="event stream password" %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L83) {{< highlight toml >}} [grpc.services.storageprovider] -events = 0 +nats_password = "event stream password" {{< /highlight >}} {{% /dir %}} From 3c8e7dfe36c002bfe2e8ddf50ed165b78fd5e257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 21 Nov 2024 11:21:56 +0100 Subject: [PATCH 6/8] Use the user id instead of username for the space root to avoid clashes --- .drone.star | 2 +- tests/oc-integration-tests/drone/storage-users-posixfs.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index b42526c117..2256771cfb 100644 --- a/.drone.star +++ b/.drone.star @@ -761,7 +761,7 @@ def posixfsIntegrationTests(parallelRuns, skipExceptParts = []): "environment": { "TEST_SERVER_URL": "http://revad-services:20080", "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "bash -cx 'for i in {1..30}; do rm -rf /drone/src/tmp/reva/data/users/* /drone/src/tmp/reva/data/indexes/by-type/* && break || sleep 5; done; sleep 1'", + "DELETE_USER_DATA_CMD": "bash -cx 'rm -rf /drone/src/tmp/reva/data/users/* /drone/src/tmp/reva/data/indexes/by-type/*'", "STORAGE_DRIVER": "ocis", "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", "TEST_WITH_LDAP": "true", diff --git a/tests/oc-integration-tests/drone/storage-users-posixfs.toml b/tests/oc-integration-tests/drone/storage-users-posixfs.toml index 212fabdafe..4813f9e284 100644 --- a/tests/oc-integration-tests/drone/storage-users-posixfs.toml +++ b/tests/oc-integration-tests/drone/storage-users-posixfs.toml @@ -24,7 +24,7 @@ root = "/drone/src/tmp/reva/data" permissionssvc = "localhost:10000" treetime_accounting = true treesize_accounting = true -personalspacepath_template = "users/{{.User.Username}}" +personalspacepath_template = "users/{{.User.Id.OpaqueId}}" generalspacepath_template = "projects/{{.SpaceId}}" watch_fs = true From df1b8428faab046985e76bce3d76fb350cb0f3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 21 Nov 2024 11:23:07 +0100 Subject: [PATCH 7/8] Use the initialized logger in decomposedfs instead of creating our own --- pkg/storage/fs/ocis/ocis.go | 4 ++-- pkg/storage/fs/posix/posix.go | 2 +- pkg/storage/fs/s3ng/s3ng.go | 4 ++-- pkg/storage/utils/decomposedfs/decomposedfs.go | 12 +++++++----- pkg/storage/utils/decomposedfs/decomposedfs_test.go | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index f8ec0a85f3..32d2766b58 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -36,7 +36,7 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. -func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (storage.FS, error) { +func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) { o, err := options.New(m) if err != nil { return nil, err @@ -47,5 +47,5 @@ func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (sto return nil, err } - return decomposedfs.NewDefault(m, bs, stream) + return decomposedfs.NewDefault(m, bs, stream, log) } diff --git a/pkg/storage/fs/posix/posix.go b/pkg/storage/fs/posix/posix.go index e6faca47b1..e5387602cd 100644 --- a/pkg/storage/fs/posix/posix.go +++ b/pkg/storage/fs/posix/posix.go @@ -134,7 +134,7 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s Trashbin: trashbin, } - dfs, err := decomposedfs.New(&o.Options, aspects) + dfs, err := decomposedfs.New(&o.Options, aspects, log) if err != nil { return nil, err } diff --git a/pkg/storage/fs/s3ng/s3ng.go b/pkg/storage/fs/s3ng/s3ng.go index d261ad0ea5..eb755127be 100644 --- a/pkg/storage/fs/s3ng/s3ng.go +++ b/pkg/storage/fs/s3ng/s3ng.go @@ -35,7 +35,7 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. -func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (storage.FS, error) { +func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) { o, err := parseConfig(m) if err != nil { return nil, err @@ -59,5 +59,5 @@ func New(m map[string]interface{}, stream events.Stream, _ *zerolog.Logger) (sto return nil, err } - return decomposedfs.NewDefault(m, bs, stream) + return decomposedfs.NewDefault(m, bs, stream, log) } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 8b95640889..97c7cf09bd 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -35,6 +35,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/jellydator/ttlcache/v2" "github.com/pkg/errors" + "github.com/rs/zerolog" tusd "github.com/tus/tusd/v2/pkg/handler" microstore "go-micro.dev/v4/store" "go.opentelemetry.io/otel" @@ -125,10 +126,12 @@ type Decomposedfs struct { userSpaceIndex *spaceidindex.Index groupSpaceIndex *spaceidindex.Index spaceTypeIndex *spaceidindex.Index + + log *zerolog.Logger } // NewDefault returns an instance with default components -func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) (storage.FS, error) { +func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream, log *zerolog.Logger) (storage.FS, error) { o, err := options.New(m) if err != nil { return nil, err @@ -169,14 +172,12 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) ( Trashbin: &DecomposedfsTrashbin{}, } - return New(o, aspects) + return New(o, aspects, log) } // New returns an implementation of the storage.FS interface that talks to // a local filesystem. -func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) { - log := logger.New() - +func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (storage.FS, error) { err := aspects.Tree.Setup() if err != nil { log.Error().Err(err).Msg("could not setup tree") @@ -235,6 +236,7 @@ func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) { userSpaceIndex: userSpaceIndex, groupSpaceIndex: groupSpaceIndex, spaceTypeIndex: spaceTypeIndex, + log: log, } fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens) if err = fs.trashbin.Setup(fs); err != nil { diff --git a/pkg/storage/utils/decomposedfs/decomposedfs_test.go b/pkg/storage/utils/decomposedfs/decomposedfs_test.go index ef7bc144fe..870c4c5efd 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs_test.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs_test.go @@ -61,7 +61,7 @@ var _ = Describe("Decomposed", func() { _, err := decomposedfs.NewDefault(map[string]interface{}{ "root": env.Root, "permissionssvc": "any", - }, bs, nil) + }, bs, nil, nil) Expect(err).ToNot(HaveOccurred()) }) }) From ad7679e7ae512b67064b48767a40e9fc62bed577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 21 Nov 2024 12:16:04 +0100 Subject: [PATCH 8/8] Fix tests --- pkg/storage/fs/ocis/ocis_test.go | 3 ++- pkg/storage/fs/posix/posix_test.go | 3 ++- pkg/storage/fs/posix/testhelpers/helpers.go | 2 +- pkg/storage/fs/s3ng/s3ng_test.go | 5 +++-- pkg/storage/utils/decomposedfs/testhelpers/helpers.go | 3 ++- pkg/storage/utils/decomposedfs/upload_async_test.go | 3 ++- pkg/storage/utils/decomposedfs/upload_test.go | 3 ++- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis_test.go b/pkg/storage/fs/ocis/ocis_test.go index db5a12bb6d..4656b142c7 100644 --- a/pkg/storage/fs/ocis/ocis_test.go +++ b/pkg/storage/fs/ocis/ocis_test.go @@ -23,6 +23,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/fs/ocis" "github.com/cs3org/reva/v2/tests/helpers" + "github.com/rs/zerolog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -53,7 +54,7 @@ var _ = Describe("Ocis", func() { Describe("New", func() { It("returns a new instance", func() { - _, err := ocis.New(options, nil, nil) + _, err := ocis.New(options, nil, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/fs/posix/posix_test.go b/pkg/storage/fs/posix/posix_test.go index 5a7e1fcc9e..c727d2a990 100644 --- a/pkg/storage/fs/posix/posix_test.go +++ b/pkg/storage/fs/posix/posix_test.go @@ -23,6 +23,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/fs/posix" "github.com/cs3org/reva/v2/tests/helpers" + "github.com/rs/zerolog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -56,7 +57,7 @@ var _ = Describe("Posix", func() { Describe("New", func() { It("returns a new instance", func() { - _, err := posix.New(options, nil, nil) + _, err := posix.New(options, nil, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/fs/posix/testhelpers/helpers.go b/pkg/storage/fs/posix/testhelpers/helpers.go index 0f00229765..39930797af 100644 --- a/pkg/storage/fs/posix/testhelpers/helpers.go +++ b/pkg/storage/fs/posix/testhelpers/helpers.go @@ -195,7 +195,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { Permissions: permissions.NewPermissions(pmock, permissionsSelector), Trashbin: tb, } - fs, err := decomposedfs.New(&o.Options, aspects) + fs, err := decomposedfs.New(&o.Options, aspects, &logger) if err != nil { return nil, err } diff --git a/pkg/storage/fs/s3ng/s3ng_test.go b/pkg/storage/fs/s3ng/s3ng_test.go index bae0b88417..515666f0db 100644 --- a/pkg/storage/fs/s3ng/s3ng_test.go +++ b/pkg/storage/fs/s3ng/s3ng_test.go @@ -23,6 +23,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng" "github.com/cs3org/reva/v2/tests/helpers" + "github.com/rs/zerolog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -58,12 +59,12 @@ var _ = Describe("S3ng", func() { Describe("New", func() { It("fails on missing s3 configuration", func() { - _, err := s3ng.New(map[string]interface{}{}, nil, nil) + _, err := s3ng.New(map[string]interface{}{}, nil, &zerolog.Logger{}) Expect(err).To(MatchError("S3 configuration incomplete")) }) It("works with complete configuration", func() { - _, err := s3ng.New(options, nil, nil) + _, err := s3ng.New(options, nil, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index b8b9d5d53b..c0502a3ea5 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -34,6 +34,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/store" "github.com/google/uuid" + "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "google.golang.org/grpc" @@ -178,7 +179,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { Permissions: permissions.NewPermissions(pmock, permissionsSelector), Trashbin: &decomposedfs.DecomposedfsTrashbin{}, } - fs, err := decomposedfs.New(o, aspects) + fs, err := decomposedfs.New(o, aspects, &zerolog.Logger{}) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index f5c0907202..4ce18be7b8 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -31,6 +31,7 @@ import ( "github.com/cs3org/reva/v2/pkg/store" "github.com/cs3org/reva/v2/pkg/utils" "github.com/cs3org/reva/v2/tests/helpers" + "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "google.golang.org/grpc" @@ -180,7 +181,7 @@ var _ = Describe("Async file uploads", Ordered, func() { EventStream: stream.Chan{pub, con}, Trashbin: &DecomposedfsTrashbin{}, } - fs, err = New(o, aspects) + fs, err = New(o, aspects, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"}) diff --git a/pkg/storage/utils/decomposedfs/upload_test.go b/pkg/storage/utils/decomposedfs/upload_test.go index a8bb593924..b35f36286b 100644 --- a/pkg/storage/utils/decomposedfs/upload_test.go +++ b/pkg/storage/utils/decomposedfs/upload_test.go @@ -48,6 +48,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/store" "github.com/cs3org/reva/v2/tests/helpers" + "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "google.golang.org/grpc" @@ -144,7 +145,7 @@ var _ = Describe("File uploads", func() { Permissions: permissions.NewPermissions(pmock, permissionsSelector), Trashbin: &decomposedfs.DecomposedfsTrashbin{}, } - fs, err = decomposedfs.New(o, aspects) + fs, err = decomposedfs.New(o, aspects, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"})