diff --git a/executor.go b/executor.go index adaa2554..0ef6c11b 100644 --- a/executor.go +++ b/executor.go @@ -102,56 +102,6 @@ func downloadFile(ctx context.Context, key string, filename string, cfg Config) return nil } -func downloadFiles(ctx context.Context, sink string, prefix string, cfg Config) error { - client, err := s3Client(ctx, cfg) - if err != nil { - return err - } - - res, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(cfg.BucketName), - Prefix: aws.String(prefix), - }) - - if err != nil { - return err - } - - for _, content := range res.Contents { - pathWithoutPrefix := strings.TrimPrefix(*content.Key, prefix) - dirPath, filename := filepath.Split(pathWithoutPrefix) - - // note this is temporary, as this code will be moved to the frontend - location := filepath.Join(sink, "files", dirPath) - if err := os.MkdirAll(location, 0755); err != nil { - return err - } - result, err := client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(cfg.BucketName), - Key: content.Key, - }) - - if err != nil { - return err - } - - file, err := os.OpenFile(filepath.Join(location, filename), os.O_CREATE|os.O_RDWR, 0644) - if err != nil { - result.Body.Close() - return err - } - - if _, err := io.Copy(file, result.Body); err != nil { - result.Body.Close() - return err - } - - result.Body.Close() - } - - return nil -} - func deleteFiles(ctx context.Context, prefix string, extraFiles []string, cfg Config) { client, err := s3Client(ctx, cfg) if err != nil { @@ -834,13 +784,6 @@ func cleanupRun(ctx context.Context, db *sql.DB, executionId int64, executionUui if err := results(ctx, db, executionId, pipeline, workflow); err != nil { log.Printf("failed to save results; err=%v", err) } - - if cfg.IsLocal { - sink := filepath.Join(pipeline.Sink, "history", executionUuid) - if err := downloadFiles(ctx, sink, s3Key, cfg); err != nil { - log.Printf("failed to download files; err=%v", err) - } - } } func cloudExecute(pipeline *zjson.Pipeline, executionId int64, executionUuid string, organization string, build bool, cfg Config, db *sql.DB, hub *Hub) { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c3b092ac..c9fddc52 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -22,6 +22,7 @@ "pino-caller": "^3.4.0", "pino-pretty": "^11.2.1", "rfdc": "^1.3.1", + "s3-sync-client": "^4.3.1", "source-map-support": "^0.5.21", "use-immer": "^0.9.0", "uuidv7": "^0.6.3" @@ -123,7 +124,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", - "dev": true, "dependencies": { "@aws-crypto/util": "^3.0.0", "@aws-sdk/types": "^3.222.0", @@ -133,14 +133,12 @@ "node_modules/@aws-crypto/crc32/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/crc32c": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz", "integrity": "sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==", - "dev": true, "dependencies": { "@aws-crypto/util": "^3.0.0", "@aws-sdk/types": "^3.222.0", @@ -150,14 +148,12 @@ "node_modules/@aws-crypto/crc32c/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/ie11-detection": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", - "dev": true, "dependencies": { "tslib": "^1.11.1" } @@ -165,14 +161,12 @@ "node_modules/@aws-crypto/ie11-detection/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/sha1-browser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz", "integrity": "sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==", - "dev": true, "dependencies": { "@aws-crypto/ie11-detection": "^3.0.0", "@aws-crypto/supports-web-crypto": "^3.0.0", @@ -186,14 +180,12 @@ "node_modules/@aws-crypto/sha1-browser/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/sha256-browser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", - "dev": true, "dependencies": { "@aws-crypto/ie11-detection": "^3.0.0", "@aws-crypto/sha256-js": "^3.0.0", @@ -208,14 +200,12 @@ "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/sha256-js": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", - "dev": true, "dependencies": { "@aws-crypto/util": "^3.0.0", "@aws-sdk/types": "^3.222.0", @@ -225,14 +215,12 @@ "node_modules/@aws-crypto/sha256-js/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/supports-web-crypto": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", - "dev": true, "dependencies": { "tslib": "^1.11.1" } @@ -240,14 +228,12 @@ "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/util": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", - "dev": true, "dependencies": { "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-utf8-browser": "^3.0.0", @@ -257,14 +243,51 @@ "node_modules/@aws-crypto/util/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, + "node_modules/@aws-sdk/abort-controller": { + "version": "3.374.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/abort-controller/-/abort-controller-3.374.0.tgz", + "integrity": "sha512-pO1pqFBdIF28ZvnJmg58Erj35RLzXsTrjvHghdc/xgtSvodFFCNrUsPg6AP3On8eiw9elpHoS4P8jMx1pHDXEw==", + "deprecated": "This package has moved to @smithy/abort-controller", + "peer": true, + "dependencies": { + "@smithy/abort-controller": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/abort-controller/node_modules/@smithy/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-5imgGUlZL4dW4YWdMYAKLmal9ny/tlenM81QZY7xYyb76z9Z/QOg7oM5Ak9HQl8QfFTlGVWwcMXl+54jroRgEQ==", + "peer": true, + "dependencies": { + "@smithy/types": "^1.2.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/abort-controller/node_modules/@smithy/types": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.2.0.tgz", + "integrity": "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==", + "peer": true, + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } }, "node_modules/@aws-sdk/client-s3": { "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.590.0.tgz", "integrity": "sha512-so+pNua0ihsHaSdskw8HCwruoYTAfYSEs3ix4GD1++83C96KaJp3udAutYiCA+84JXg9zitFa7eK7ORJAVZmTw==", - "dev": true, "dependencies": { "@aws-crypto/sha1-browser": "3.0.0", "@aws-crypto/sha256-browser": "3.0.0", @@ -333,7 +356,6 @@ "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "dev": true, "dependencies": { "@aws-crypto/sha256-browser": "3.0.0", "@aws-crypto/sha256-js": "3.0.0", @@ -382,7 +404,6 @@ "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.590.0.tgz", "integrity": "sha512-3yCLPjq6WFfDpdUJKk/gSz4eAPDTjVknXaveMPi2QoVBCshneOnJsV16uNKlpVF1frTHrrDRfKYmbaVh6nFBvQ==", - "dev": true, "dependencies": { "@aws-crypto/sha256-browser": "3.0.0", "@aws-crypto/sha256-js": "3.0.0", @@ -433,7 +454,6 @@ "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "dev": true, "dependencies": { "@aws-crypto/sha256-browser": "3.0.0", "@aws-crypto/sha256-js": "3.0.0", @@ -484,7 +504,6 @@ "version": "3.588.0", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.588.0.tgz", "integrity": "sha512-O1c2+9ce46Z+iiid+W3iC1IvPbfIo5ev9CBi54GdNB9SaI8/3+f8MJcux0D6c9toCF0ArMersN/gp8ek57e9uQ==", - "dev": true, "dependencies": { "@smithy/core": "^2.1.1", "@smithy/protocol-http": "^4.0.0", @@ -502,7 +521,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/property-provider": "^3.1.0", @@ -517,7 +535,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/fetch-http-handler": "^3.0.1", @@ -537,7 +554,6 @@ "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "dev": true, "dependencies": { "@aws-sdk/credential-provider-env": "3.587.0", "@aws-sdk/credential-provider-http": "3.587.0", @@ -562,7 +578,6 @@ "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "dev": true, "dependencies": { "@aws-sdk/credential-provider-env": "3.587.0", "@aws-sdk/credential-provider-http": "3.587.0", @@ -585,7 +600,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/property-provider": "^3.1.0", @@ -601,7 +615,6 @@ "version": "3.590.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "dev": true, "dependencies": { "@aws-sdk/client-sso": "3.590.0", "@aws-sdk/token-providers": "3.587.0", @@ -619,7 +632,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/property-provider": "^3.1.0", @@ -660,7 +672,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.587.0.tgz", "integrity": "sha512-HkFXLPl8pr6BH/Q0JpOESqEKL0ZK3sk7aSZ1S6GE4RXET7H5R94THULXqQFZzD48gZcyFooO/yNKZTqrZFaWKg==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@aws-sdk/util-arn-parser": "3.568.0", @@ -678,7 +689,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.577.0.tgz", "integrity": "sha512-6dPp8Tv4F0of4un5IAyG6q++GrRrNQQ4P2NAMB1W0VO4JoEu1C8GievbbDLi88TFIFmtKpnHB0ODCzwnoe8JsA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/protocol-http": "^4.0.0", @@ -693,7 +703,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.587.0.tgz", "integrity": "sha512-URMwp/budDvKhIvZ4a6zIBfFTun/iDlPWXqsGKYjEtHt8jz27OSjCZtDtIeqW4WTBdKL8KZgQcl+DdaE5M1qiQ==", - "dev": true, "dependencies": { "@aws-crypto/crc32": "3.0.0", "@aws-crypto/crc32c": "3.0.0", @@ -712,7 +721,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/protocol-http": "^4.0.0", @@ -727,7 +735,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.577.0.tgz", "integrity": "sha512-DKPTD2D2s+t2QUo/IXYtVa/6Un8GZ+phSTBkyBNx2kfZz4Kwavhl/JJzSqTV3GfCXkVdFu7CrjoX7BZ6qWeTUA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/types": "^3.0.0", @@ -741,7 +748,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/types": "^3.0.0", @@ -755,7 +761,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/protocol-http": "^4.0.0", @@ -770,7 +775,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.587.0.tgz", "integrity": "sha512-vtXTGEiw1E9Fax4LmcU2Z208gbrC8ShrdsSLmGcRPpu5NPOGBFBSDG5sy5EDNClrFxIl/Le8coQnD0EDBtx+uQ==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@aws-sdk/util-arn-parser": "3.568.0", @@ -790,7 +794,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.587.0.tgz", "integrity": "sha512-tiZaTDj4RvhXGRAlncFn7CSEfL3iNPO67WSaxAq+Ls5j1VgczPhu5262cWONNoMgth3nXR1hhLC4ITSl/a6AzA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/property-provider": "^3.1.0", @@ -808,7 +811,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.577.0.tgz", "integrity": "sha512-i2BPJR+rp8xmRVIGc0h1kDRFcM2J9GnClqqpc+NLSjmYadlcg4mPklisz9HzwFVcRPJ5XcGf3U4BYs5G8+iTyg==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/types": "^3.0.0", @@ -822,7 +824,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@aws-sdk/util-endpoints": "3.587.0", @@ -838,7 +839,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/node-config-provider": "^3.1.0", @@ -855,7 +855,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.587.0.tgz", "integrity": "sha512-TR9+ZSjdXvXUz54ayHcCihhcvxI9W7102J1OK6MrLgBlPE7uRhAx42BR9L5lLJ86Xj3LuqPWf//o9d/zR9WVIg==", - "dev": true, "dependencies": { "@aws-sdk/middleware-sdk-s3": "3.587.0", "@aws-sdk/types": "3.577.0", @@ -872,7 +871,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/property-provider": "^3.1.0", @@ -891,7 +889,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -904,7 +901,6 @@ "version": "3.568.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -916,7 +912,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/types": "^3.0.0", @@ -931,7 +926,6 @@ "version": "3.568.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -943,7 +937,6 @@ "version": "3.577.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/types": "^3.0.0", @@ -955,7 +948,6 @@ "version": "3.587.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "dev": true, "dependencies": { "@aws-sdk/types": "3.577.0", "@smithy/node-config-provider": "^3.1.0", @@ -978,7 +970,6 @@ "version": "3.259.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", - "dev": true, "dependencies": { "tslib": "^2.3.1" } @@ -987,7 +978,6 @@ "version": "3.575.0", "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.575.0.tgz", "integrity": "sha512-cWgAwmbFYNCFzPwxL705+lWps0F3ZvOckufd2KKoEZUmtpVw9/txUXNrPySUXSmRTSRhoatIMABNfStWR043bQ==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4012,7 +4002,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4025,7 +4014,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", - "dev": true, "dependencies": { "tslib": "^2.6.2" } @@ -4034,7 +4022,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", - "dev": true, "dependencies": { "@smithy/util-base64": "^3.0.0", "tslib": "^2.6.2" @@ -4044,7 +4031,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "dev": true, "dependencies": { "@smithy/node-config-provider": "^3.1.0", "@smithy/types": "^3.0.0", @@ -4060,7 +4046,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.0.tgz", "integrity": "sha512-ygLZSSKgt9bR8HAxR9mK+U5obvAJBr6zlQuhN5soYWx/amjDoQN4dTkydTypgKe6rIbUjTILyLU+W5XFwXr4kg==", - "dev": true, "dependencies": { "@smithy/middleware-endpoint": "^3.0.1", "@smithy/middleware-retry": "^3.0.3", @@ -4079,7 +4064,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "dev": true, "dependencies": { "@smithy/node-config-provider": "^3.1.0", "@smithy/property-provider": "^3.1.0", @@ -4095,7 +4079,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.0.0.tgz", "integrity": "sha512-PUtyEA0Oik50SaEFCZ0WPVtF9tz/teze2fDptW6WRXl+RrEenH8UbEjudOz8iakiMl3lE3lCVqYf2Y+znL8QFQ==", - "dev": true, "dependencies": { "@aws-crypto/crc32": "3.0.0", "@smithy/types": "^3.0.0", @@ -4107,7 +4090,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.0.tgz", "integrity": "sha512-NB7AFiPN4NxP/YCAnrvYR18z2/ZsiHiF7VtG30gshO9GbFrIb1rC8ep4NGpJSWrz6P64uhPXeo4M0UsCLnZKqw==", - "dev": true, "dependencies": { "@smithy/eventstream-serde-universal": "^3.0.0", "@smithy/types": "^3.0.0", @@ -4121,7 +4103,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.0.tgz", "integrity": "sha512-RUQG3vQ3LX7peqqHAbmayhgrF5aTilPnazinaSGF1P0+tgM3vvIRWPHmlLIz2qFqB9LqFIxditxc8O2Z6psrRw==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4134,7 +4115,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.0.tgz", "integrity": "sha512-baRPdMBDMBExZXIUAoPGm/hntixjt/VFpU6+VmCyiYJYzRHRxoaI1MN+5XE+hIS8AJ2GCHLMFEIOLzq9xx1EgQ==", - "dev": true, "dependencies": { "@smithy/eventstream-serde-universal": "^3.0.0", "@smithy/types": "^3.0.0", @@ -4148,7 +4128,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.0.tgz", "integrity": "sha512-HNFfShmotWGeAoW4ujP8meV9BZavcpmerDbPIjkJbxKbN8RsUcpRQ/2OyIxWNxXNH2GWCAxuSB7ynmIGJlQ3Dw==", - "dev": true, "dependencies": { "@smithy/eventstream-codec": "^3.0.0", "@smithy/types": "^3.0.0", @@ -4162,7 +4141,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dev": true, "dependencies": { "@smithy/protocol-http": "^4.0.0", "@smithy/querystring-builder": "^3.0.0", @@ -4175,7 +4153,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.0.0.tgz", "integrity": "sha512-/Wbpdg+bwJvW7lxR/zpWAc1/x/YkcqguuF2bAzkJrvXriZu1vm8r+PUdE4syiVwQg7PPR2dXpi3CLBb9qRDaVQ==", - "dev": true, "dependencies": { "@smithy/chunked-blob-reader": "^3.0.0", "@smithy/chunked-blob-reader-native": "^3.0.0", @@ -4187,7 +4164,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "@smithy/util-buffer-from": "^3.0.0", @@ -4202,7 +4178,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.0.0.tgz", "integrity": "sha512-J0i7de+EgXDEGITD4fxzmMX8CyCNETTIRXlxjMiNUvvu76Xn3GJ31wQR85ynlPk2wI1lqoknAFJaD1fiNDlbIA==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "@smithy/util-utf8": "^3.0.0", @@ -4216,7 +4191,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4226,7 +4200,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -4238,7 +4211,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.0.tgz", "integrity": "sha512-Tm0vrrVzjlD+6RCQTx7D3Ls58S3FUH1ZCtU1MIh/qQmaOo1H9lMN2as6CikcEwgattnA9SURSdoJJ27xMcEfMA==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "@smithy/util-utf8": "^3.0.0", @@ -4249,7 +4221,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "dev": true, "dependencies": { "@smithy/protocol-http": "^4.0.0", "@smithy/types": "^3.0.0", @@ -4263,7 +4234,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dev": true, "dependencies": { "@smithy/middleware-serde": "^3.0.0", "@smithy/node-config-provider": "^3.1.0", @@ -4281,7 +4251,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dev": true, "dependencies": { "@smithy/node-config-provider": "^3.1.0", "@smithy/protocol-http": "^4.0.0", @@ -4301,7 +4270,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4314,7 +4282,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4327,7 +4294,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dev": true, "dependencies": { "@smithy/property-provider": "^3.1.0", "@smithy/shared-ini-file-loader": "^3.1.0", @@ -4342,7 +4308,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dev": true, "dependencies": { "@smithy/abort-controller": "^3.0.0", "@smithy/protocol-http": "^4.0.0", @@ -4358,7 +4323,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4371,7 +4335,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4384,7 +4347,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "@smithy/util-uri-escape": "^3.0.0", @@ -4398,7 +4360,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4411,7 +4372,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0" }, @@ -4423,7 +4383,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4436,7 +4395,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dev": true, "dependencies": { "@smithy/is-array-buffer": "^3.0.0", "@smithy/types": "^3.0.0", @@ -4454,7 +4412,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dev": true, "dependencies": { "@smithy/middleware-endpoint": "^3.0.1", "@smithy/middleware-stack": "^3.0.0", @@ -4471,7 +4428,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -4483,7 +4439,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dev": true, "dependencies": { "@smithy/querystring-parser": "^3.0.0", "@smithy/types": "^3.0.0", @@ -4494,7 +4449,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dev": true, "dependencies": { "@smithy/util-buffer-from": "^3.0.0", "@smithy/util-utf8": "^3.0.0", @@ -4508,7 +4462,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "dev": true, "dependencies": { "tslib": "^2.6.2" } @@ -4517,7 +4470,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -4529,7 +4481,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dev": true, "dependencies": { "@smithy/is-array-buffer": "^3.0.0", "tslib": "^2.6.2" @@ -4542,7 +4493,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -4554,7 +4504,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "dev": true, "dependencies": { "@smithy/property-provider": "^3.1.0", "@smithy/smithy-client": "^3.1.1", @@ -4570,7 +4519,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "dev": true, "dependencies": { "@smithy/config-resolver": "^3.0.1", "@smithy/credential-provider-imds": "^3.1.0", @@ -4588,7 +4536,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.1.tgz", "integrity": "sha512-ZRT0VCOnKlVohfoABMc8lWeQo/JEFuPWctfNRXgTHbyOVssMOLYFUNWukxxiHRGVAhV+n3c0kPW+zUqckjVPEA==", - "dev": true, "dependencies": { "@smithy/node-config-provider": "^3.1.0", "@smithy/types": "^3.0.0", @@ -4602,7 +4549,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -4614,7 +4560,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dev": true, "dependencies": { "@smithy/types": "^3.0.0", "tslib": "^2.6.2" @@ -4627,7 +4572,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dev": true, "dependencies": { "@smithy/service-error-classification": "^3.0.0", "@smithy/types": "^3.0.0", @@ -4641,7 +4585,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dev": true, "dependencies": { "@smithy/fetch-http-handler": "^3.0.1", "@smithy/node-http-handler": "^3.0.0", @@ -4660,7 +4603,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dev": true, "dependencies": { "tslib": "^2.6.2" }, @@ -4672,7 +4614,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dev": true, "dependencies": { "@smithy/util-buffer-from": "^3.0.0", "tslib": "^2.6.2" @@ -4685,7 +4626,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.0.0.tgz", "integrity": "sha512-+fEXJxGDLCoqRKVSmo0auGxaqbiCo+8oph+4auefYjaNxjOLKSY2MxVQfRzo65PaZv4fr+5lWg+au7vSuJJ/zw==", - "dev": true, "dependencies": { "@smithy/abort-controller": "^3.0.0", "@smithy/types": "^3.0.0", @@ -6248,8 +6188,7 @@ "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", - "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", - "dev": true + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" }, "node_modules/brace-expansion": { "version": "2.0.1", @@ -8983,7 +8922,6 @@ "version": "4.2.5", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", - "dev": true, "funding": [ { "type": "paypal", @@ -12869,6 +12807,18 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/s3-sync-client": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/s3-sync-client/-/s3-sync-client-4.3.1.tgz", + "integrity": "sha512-nWbbKCNnXmWvD8XwdWhX25VNxIhgQEm6vXqSYjwyBNZI07OuMOr/LNOYmEPcLfqFFjy55ZNcFSBI18W29ybuUw==", + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/abort-controller": "^3.x.x", + "@aws-sdk/client-s3": "^3.x.x" + } + }, "node_modules/safe-array-concat": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", @@ -13587,8 +13537,7 @@ "node_modules/strnum": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", - "dev": true + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" }, "node_modules/style-mod": { "version": "4.1.2", @@ -14374,7 +14323,6 @@ "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" diff --git a/frontend/package.json b/frontend/package.json index d15e25c3..592de334 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -41,6 +41,7 @@ "pino-caller": "^3.4.0", "pino-pretty": "^11.2.1", "rfdc": "^1.3.1", + "s3-sync-client": "^4.3.1", "source-map-support": "^0.5.21", "use-immer": "^0.9.0", "uuidv7": "^0.6.3" diff --git a/frontend/server/execution.js b/frontend/server/execution.js new file mode 100644 index 00000000..39b9fedf --- /dev/null +++ b/frontend/server/execution.js @@ -0,0 +1,14 @@ +import path from "path"; +import { syncS3ToLocalDirectory } from "./s3"; + +export async function syncExecutionResults( + buffer, + pipelineUuid, + executionUuid, + anvilConfiguration, +) { + const s3Prefix = `${pipelineUuid}/${executionUuid}`; + const localPath = path.join(buffer, "history", executionUuid, "files"); + + await syncS3ToLocalDirectory(s3Prefix, localPath, anvilConfiguration); +} diff --git a/frontend/server/router.ts b/frontend/server/router.ts index 77ce5709..0f767c8a 100644 --- a/frontend/server/router.ts +++ b/frontend/server/router.ts @@ -20,6 +20,8 @@ import { import { publicProcedure, router } from "./trpc"; import { errorHandling } from "./middleware"; import { logger } from "./logger.js"; +import { anvilConfigurationSchema } from "./schema"; +import { syncExecutionResults } from "./execution"; export const appRouter = router({ getBlocks: publicProcedure.use(errorHandling).query(async () => { @@ -224,22 +226,7 @@ export const appRouter = router({ buffer: z.string(), name: z.string(), rebuild: z.boolean(), - anvilConfiguration: z.object({ - name: z.string(), - anvil: z.object({ - host: z.string(), - port: z.string(), - token: z.string(), - }), - s3: z.object({ - host: z.string(), - port: z.string(), - region: z.string(), - bucket: z.string(), - accessKeyId: z.string(), - secretAccessKey: z.string(), - }), - }), + anvilConfiguration: anvilConfigurationSchema, }), ) .mutation(async (opts) => { @@ -293,6 +280,27 @@ export const appRouter = router({ return await saveBlockSpecs(blockPath, blockSpecs); }), + downloadExecutionResults: publicProcedure + .use(errorHandling) + .input( + z.object({ + buffer: z.string(), + pipelineUuid: z.string(), + executionUuid: z.string(), + anvilConfiguration: anvilConfigurationSchema, + }), + ) + .mutation(async (opts) => { + const { input } = opts; + const { buffer, pipelineUuid, executionUuid, anvilConfiguration } = input; + + await syncExecutionResults( + buffer, + pipelineUuid, + executionUuid, + anvilConfiguration, + ); + }), runTest: publicProcedure .use(errorHandling) .input( diff --git a/frontend/server/s3.js b/frontend/server/s3.js index d43a89d2..215764ee 100644 --- a/frontend/server/s3.js +++ b/frontend/server/s3.js @@ -1,14 +1,15 @@ import { + CopyObjectCommand, + GetObjectCommand, HeadObjectCommand, PutObjectCommand, - GetObjectCommand, - CopyObjectCommand, S3Client, } from "@aws-sdk/client-s3"; import fs from "fs/promises"; +import path from "path"; +import { S3SyncClient } from "s3-sync-client"; import config from "../config"; import { getDirectoryFilesRecursive } from "./fileSystem"; -import path from "path"; import { logger } from "./logger"; function getClient(configuration) { @@ -183,3 +184,15 @@ export async function getFile(key, destinationPath, anvilConfiguration) { throw new Error(message); } } + +export async function syncS3ToLocalDirectory( + s3Prefix, + localPath, + anvilConfiguration, +) { + const client = getClient(anvilConfiguration); + const { sync } = new S3SyncClient({ client: client }); + + const s3Path = `s3://${anvilConfiguration.s3.bucket}/${s3Prefix}`; + await sync(s3Path, localPath); +} diff --git a/frontend/server/schema.js b/frontend/server/schema.js new file mode 100644 index 00000000..7f305b0b --- /dev/null +++ b/frontend/server/schema.js @@ -0,0 +1,17 @@ +import { z } from "zod"; + +export const anvilConfigurationSchema = z.object({ + name: z.string(), + anvil: z.object({ + host: z.string(), + port: z.string(), + }), + s3: z.object({ + host: z.string(), + port: z.string(), + region: z.string(), + bucket: z.string(), + accessKeyId: z.string(), + secretAccessKey: z.string(), + }), +}); diff --git a/frontend/src/components/ui/ExecutionDataGrid.jsx b/frontend/src/components/ui/ExecutionDataGrid.jsx index 0ba80ed3..f031fa91 100644 --- a/frontend/src/components/ui/ExecutionDataGrid.jsx +++ b/frontend/src/components/ui/ExecutionDataGrid.jsx @@ -15,19 +15,22 @@ import { import { PipelineStopButton } from "./PipelineStopButton"; import { useState, useEffect } from "react"; import { activeConfigurationAtom } from "@/atoms/anvilConfigurationsAtom"; +import { useSyncExecutionResults } from "@/hooks/useExecutionResults"; export const ExecutionDataGrid = ({ executions, closeModal }) => { const [workspace, setWorkspace] = useImmerAtom(workspaceAtom); const [pipelineList, setPipelineList] = useState([]); const [configuration] = useAtom(activeConfigurationAtom); + const syncResults = useSyncExecutionResults(); - const selectPipeline = (pipeline) => { + const selectPipeline = async (pipeline) => { const key = pipeline.id + "." + pipeline.record.Execution; setWorkspace((draft) => { draft.tabs[key] = {}; draft.active = key; }); + await syncResults(key); closeModal(); }; diff --git a/frontend/src/components/ui/SocketFetcher.jsx b/frontend/src/components/ui/SocketFetcher.jsx index 9ef8fd64..e8895552 100644 --- a/frontend/src/components/ui/SocketFetcher.jsx +++ b/frontend/src/components/ui/SocketFetcher.jsx @@ -1,10 +1,11 @@ -import { useAtom } from "jotai"; +import { parseLogLine } from "@/atoms/logsAtom"; import { pipelineAtom } from "@/atoms/pipelineAtom"; -import { useEffect, useCallback } from "react"; +import { useSyncExecutionResults } from "@/hooks/useExecutionResults"; import { useStableWebSocket } from "@/hooks/useStableWebsocket"; import { useUnifiedLogs } from "@/hooks/useUnifiedLogs"; -import { parseLogLine } from "@/atoms/logsAtom"; import { enableMapSet } from "immer"; +import { useAtom } from "jotai"; +import { useCallback, useEffect } from "react"; enableMapSet(); @@ -13,6 +14,7 @@ export default function SocketFetcher() { const { lastMessage, readyState, wsError } = useStableWebSocket( pipeline?.socketUrl, ); + const syncResults = useSyncExecutionResults(); const { updateLogs } = useUnifiedLogs(); @@ -32,6 +34,11 @@ export default function SocketFetcher() { } } }); + + if (parsedLogEntry?.event?.tag === "outputs") { + const key = `${pipeline.record.Uuid}.${pipeline.record.Execution}`; + syncResults(key); + } }); useEffect(() => { diff --git a/frontend/src/hooks/useExecutionResults.jsx b/frontend/src/hooks/useExecutionResults.jsx new file mode 100644 index 00000000..cfe4570e --- /dev/null +++ b/frontend/src/hooks/useExecutionResults.jsx @@ -0,0 +1,23 @@ +import { workspaceAtom } from "@/atoms/pipelineAtom"; +import { activeConfigurationAtom } from "@/atoms/anvilConfigurationsAtom"; +import { trpc } from "@/utils/trpc"; +import { useAtom } from "jotai"; + +export const useSyncExecutionResults = () => { + const [workspace] = useAtom(workspaceAtom); + const [configuration] = useAtom(activeConfigurationAtom); + const downloadExecutionResults = trpc.downloadExecutionResults.useMutation(); + + const syncExecutionResults = async (key) => { + console.log("syncin"); + const pipeline = workspace.pipelines[key]; + await downloadExecutionResults.mutateAsync({ + buffer: pipeline.buffer, + pipelineUuid: pipeline.record.Uuid, + executionUuid: pipeline.record.Execution, + anvilConfiguration: configuration, + }); + }; + + return syncExecutionResults; +};