B/multi files (#241)

* checkpoint very broken version * update * mostly working * more changes * use indexed db for cache instead of local storage * fix package lock * fix the tabs * fix accidently removed var * fix linter complaints * package-lock * must please the all powerful linter * fix comments * fixing multifiles * fixing multi file and folder * fixing folder * remove unused useEffect
zetane · Nov 4, 2024 · 09e0fe3 · 09e0fe3
1 parent 6795bb3
commit 09e0fe3
Show file tree

Hide file tree

Showing 7 changed files with 144 additions and 73 deletions.
diff --git a/frontend/server/pipelineSerialization.js b/frontend/server/pipelineSerialization.js
@@ -221,52 +221,81 @@ async function uploadBlocks(
             param.value = `"${fileName}"`;
             param.type = "blob";
           }
-        } else if (param.type === "folder" || param.type === "file[]") {
-          try {
-            const cleanedValue = param.value.replace(/\\/g, "\\\\");
-            const fileNames = [];
-            const filePaths = JSON.parse(cleanedValue);
-            const firstFilePath = filePaths[0];
-            const pathSegments = firstFilePath.split(/[/\\]/);
-            const rootFolder = pathSegments[pathSegments.length - 2];
-
-            for (const filePath of filePaths) {
-              // console.log("Uploading file:", filePath); // Debugging log
-              let relativePath = filePath.split(rootFolder)[1];
-
-              if (param.type === "folder") {
-                relativePath = relativePath.replace(/\\/g, "/").trim();
-              } else {
-                relativePath = relativePath.replace(/\\/g, "").trim();
-              }
-
-              fileNames.push(relativePath);
-              const awsKey = `${pipelineId}/${executionId}/${relativePath}`;
-
+        } else if (param.type == "file[]") {
+          const files = param.value;
+          const uploaded = await Promise.all(
+            files.map(async (filePath) => {
+              const fileName = path.basename(filePath);
+              const awsKey = `${pipelineId}/${executionId}/${fileName}`;
               if (filePath && filePath.trim()) {
                 await checkAndUpload(awsKey, filePath, anvilConfiguration);
-                // console.log(`Uploaded: ${relativePath} to ${awsKey}`); // which file uploaded.
-              } else {
-                // log invalid paths.
-                console.error("Invalid file path:", filePath);
               }
-            }
-            // Preserve the original value and update type
-            param.value =
-              fileNames.length > 0
-                ? `["${fileNames.join('", "')}"]`
-                : param.value;
-            param.type = "blob";
-          } catch (error) {
-            console.error("Error processing folder:", error);
+              return fileName;
+            }),
+          );
+
+          param.value = JSON.stringify(uploaded);
+          param.type = "blob[]";
+        } else if (param.type === "folder") {
+          const files = param?.value ?? [];
+          const folder = parameters?.folderName?.value;
+          if (!folder || folder == "") {
+            throw new Error("Folder block must set a folderName parameter");
           }
+          const uploaded = await Promise.all(
+            files.map(async (filePath) => {
+              const folderIndex = filePath.indexOf(folder);
+              if (folderIndex === -1) {
+                throw new Error("Folder block must set a folderName parameter");
+              }
+              const slicePath = filePath.slice(folderIndex);
+              const awsKey = `${pipelineId}/${executionId}/${slicePath}`;
+              if (filePath && filePath.trim()) {
+                await checkAndUpload(awsKey, filePath, anvilConfiguration);
+              }
+              return slicePath;
+            }),
+          );
+          param.value = JSON.stringify(uploaded);
+          param.type = "folderBlob";
         } else if (param.type == "blob") {
           const copyKey = param.value;
           const fileName = param.value.split("/").at(-1);
           const newAwsKey = `${pipelineId}/${executionId}/${fileName}`;
-
           await checkAndCopy(newAwsKey, copyKey, anvilConfiguration);
           param.value = `"${fileName}"`;
+        } else if (param.type == "blob[]") {
+          const s3files = param.value;
+          const uploadedFiles = await Promise.all(
+            s3files.map(async (s3file) => {
+              const fileName = s3file.split("/").at(-1);
+              const newAwsKey = `${pipelineId}/${executionId}/${fileName}`;
+              await checkAndCopy(newAwsKey, s3file, anvilConfiguration);
+              return fileName;
+            }),
+          );
+
+          param.value = JSON.stringify(uploadedFiles);
+          param.type = "blob[]";
+        } else if (param.type == "folderBlob") {
+          const folder = parameters?.folderName?.value;
+          if (!folder || folder == "") {
+            throw new Error("Folder block must set a folderName parameter");
+          }
+          const awsPaths = param?.value ?? [];
+          const uploadedFiles = await Promise.all(
+            awsPaths.map(async (s3file) => {
+              const folderIndex = s3file.indexOf(folder);
+              if (folderIndex === -1) {
+                throw new Error("Folder block must set a folderName parameter");
+              }
+              const slicePath = s3file.slice(folderIndex);
+              const newAwsKey = `${pipelineId}/${executionId}/${slicePath}`;
+              await checkAndCopy(newAwsKey, s3file, anvilConfiguration);
+              return slicePath;
+            }),
+          );
+          param.value = JSON.stringify(uploadedFiles);
         }
       }
     }

diff --git a/frontend/src/components/ui/blockGenerator/BlockGenerator.jsx b/frontend/src/components/ui/blockGenerator/BlockGenerator.jsx
@@ -4,7 +4,7 @@ import { Code, View, CloudLogging } from "@carbon/icons-react";
 import { useImmerAtom } from "jotai-immer";
 import { useEffect, useRef, useState, useMemo } from "react";
 import { FileBlock } from "./FileBlock";
-import { FolderBlock } from "./Folder-uploadBlock";
+import { FolderBlock } from "./FolderBlock";
 import { MultiFileBlock } from "./MultiFileBlock";
 import { modalContentAtom } from "@/atoms/modalAtom";
 import { useAtom } from "jotai";
@@ -139,7 +139,7 @@ const BlockGenerator = ({
   const type =
     block?.action?.parameters?.path?.type ||
     block?.action?.parameters?.files?.type;
-  if (type == "folder" || block.information.id == "folder-upload") {
+  if (block?.information?.id == "folder-upload") {
     content = (
       <FolderBlock
         blockId={id}
@@ -148,11 +148,7 @@ const BlockGenerator = ({
         history={history}
       />
     );
-  } else if (
-    type == "file[]" ||
-    type == "multiFile" ||
-    block.information.id == "multi-file-upload"
-  ) {
+  } else if (type == "file[]" || type == "blob[]") {
     content = (
       <MultiFileBlock
         blockId={id}

diff --git a/.../ui/blockGenerator/Folder-uploadBlock.jsx → ...ponents/ui/blockGenerator/FolderBlock.jsx b/.../ui/blockGenerator/Folder-uploadBlock.jsx → ...ponents/ui/blockGenerator/FolderBlock.jsx
@@ -1,30 +1,39 @@
 import { useEffect, useRef, useState } from "react";
+import { TreeView, TreeNode } from "@carbon/react";
+import { Folder, Document } from "@carbon/icons-react";
 
 export const FolderBlock = ({ blockId, block, setFocusAction, history }) => {
   const fileInput = useRef();
   const [renderPath, setRenderPath] = useState(null);
   const [folderName, setFolderName] = useState(
     block?.action?.parameters["folderName"]?.value || null,
   );
-  const [filePaths, setFilePaths] = useState(() => {
-    const existingPaths = block?.action?.parameters["path"]?.value || "[]";
-    return JSON.parse(existingPaths);
-  });
+  const [isExpanded, setIsExpanded] = useState(false);
 
   useEffect(() => {
-    if (history && filePaths.length > 0 && folderName) {
-      const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`;
+    const type = block?.action?.parameters["path"]?.type;
+    const value = block?.action?.parameters["path"]?.value;
+    if (history && type == "folderBlob" && typeof value == "string") {
+      const paths = JSON.parse(value);
+      const s3Files = paths.map((name) => {
+        return history + "/" + name;
+      });
       setFocusAction((draft) => {
-        draft.data[blockId].action.parameters["path"].type = "folder";
-        draft.data[blockId].action.parameters["path"].value = formattedValue;
+        draft.data[blockId].action.parameters["path"].type = "folderBlob";
+        draft.data[blockId].action.parameters["path"].value = s3Files;
         draft.data[blockId].action.parameters["folderName"] = {
           type: "string",
           value: folderName,
         };
       });
-      setRenderPath(folderName);
+
+      const folderInfo = {
+        files: paths,
+        folderName: folderName,
+      };
+      setRenderPath(folderInfo);
     }
-  }, [block, folderName, filePaths]);
+  }, [block, folderName]);
 
   const processFiles = (files) => {
     const filePaths = [];
@@ -35,7 +44,11 @@ export const FolderBlock = ({ blockId, block, setFocusAction, history }) => {
         if (file.webkitDirectory) {
           readDirectory(file.children, fullPath);
         } else {
-          filePaths.push(fullPath);
+          const baseName = file.path.split("/").at(-1);
+          // DS STORE I BANISH THEE
+          if (baseName != ".DS_Store") {
+            filePaths.push(fullPath);
+          }
         }
       }
     };
@@ -46,30 +59,63 @@ export const FolderBlock = ({ blockId, block, setFocusAction, history }) => {
   const loadFiles = () => {
     const files = Array.from(fileInput.current.files);
     const filePaths = processFiles(files);
-    const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`;
 
     const firstFilePath = filePaths[0];
     const pathSegments = firstFilePath.split(/[/\\]/);
     const extractedFolderName = pathSegments[pathSegments.length - 2];
 
     setFolderName(extractedFolderName);
-    setFilePaths(filePaths);
 
     setFocusAction((draft) => {
-      draft.data[blockId].action.parameters["path"].value = formattedValue;
+      draft.data[blockId].action.parameters["path"].value = filePaths;
       draft.data[blockId].action.parameters["path"].type = "folder";
       draft.data[blockId].action.parameters["folderName"] = {
         type: "string",
         value: extractedFolderName,
       };
     });
 
-    setRenderPath(extractedFolderName);
+    const folderInfo = {
+      files: filePaths,
+      folderName: extractedFolderName,
+    };
+
+    setRenderPath(folderInfo);
   };
 
+  let treeView = null;
+  if (renderPath?.folderName) {
+    treeView = (
+      <TreeView>
+        <TreeNode
+          key={0}
+          isExpanded={isExpanded}
+          onToggle={() => setIsExpanded(!isExpanded)}
+          label={renderPath.folderName}
+          renderIcon={Folder}
+          value={renderPath.folderName}
+        >
+          {renderPath.files.map((filePath, index) => (
+            <TreeNode
+              key={index + 1}
+              label={filePath}
+              value={filePath}
+              renderIcon={Document}
+              isSelectable={false}
+              onSelect={(e) => {
+                e.preventDefault();
+                e.stopPropagation();
+              }}
+            />
+          ))}
+        </TreeNode>
+      </TreeView>
+    );
+  }
+
   return (
     <div className="block-content">
-      <div className="mb-2 pl-2">{renderPath}</div>
+      {treeView}
       <input
         id="folder-block"
         type="file"

diff --git a/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx b/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx
@@ -5,20 +5,20 @@ export const MultiFileBlock = ({ blockId, block, setFocusAction, history }) => {
   const [renderedFiles, setRenderedFiles] = useState([]);
 
   useEffect(() => {
-    if (history) {
-      const fileNames = block?.action?.parameters["files"]?.value.split(", ");
-      const updatedFiles = fileNames.map((fileName) => {
-        const fileSplit = fileName.split("/");
-        return fileSplit.length > 1 ? fileSplit.at(-1) : fileName;
+    const type = block?.action?.parameters["files"]?.type;
+    const value = block?.action?.parameters["files"]?.value;
+    if (history && type != "file[]" && typeof value == "string") {
+      const fileNames = JSON.parse(value);
+      const s3Files = fileNames.map((name) => {
+        return history + "/" + name;
       });
 
       setFocusAction((draft) => {
-        draft.data[blockId].action.parameters["files"].type = "file[]";
-        draft.data[blockId].action.parameters["files"].value =
-          fileNames.join(", ");
+        draft.data[blockId].action.parameters["files"].type = "blob[]";
+        draft.data[blockId].action.parameters["files"].value = s3Files;
       });
 
-      setRenderedFiles(updatedFiles);
+      setRenderedFiles(fileNames);
     }
   }, [block]);
 
@@ -42,9 +42,8 @@ export const MultiFileBlock = ({ blockId, block, setFocusAction, history }) => {
   const loadFiles = () => {
     const files = Array.from(fileInput.current.files);
     const filePaths = processFiles(files);
-    const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`;
     setFocusAction((draft) => {
-      draft.data[blockId].action.parameters["files"].value = formattedValue; // Update to formatted value
+      draft.data[blockId].action.parameters["files"].value = filePaths; // Update to formatted value
       draft.data[blockId].action.parameters["files"].type = "file[]";
     });
 

diff --git a/frontend/src/styles/color-tokens.scss b/frontend/src/styles/color-tokens.scss
@@ -29,7 +29,8 @@
   --cds-layer: var(--ztn-gray-2) !important;
   --cds-layer-accent: var(--ztn-white-0) !important;
   --cds-text-primary: var(--ztn-black-0) !important;
-  --focus-border: var(--ztn-purple-0);
+  --focus-border: var(--ztn-purple-0) !important;
+  --cds-interactive: var(--ztn-purple-0) !important;
 }
 
 @mixin ztn-colors-g100 {
@@ -63,4 +64,5 @@
   --pipeline-background-color: var(--ztn-gray-1) !important;
   --focus-border: var(--ztn-purple-0);
   --cds-text-primary: var(--ztn-white-0) !important;
+  --cds-interactive: var(--ztn-purple-0) !important;
 }
diff --git a/frontend/src/styles/drawflow.scss b/frontend/src/styles/drawflow.scss
@@ -62,7 +62,6 @@
 
 .drawflow svg {
   z-index: 0;
-  position: absolute;
   overflow: visible !important;
   background: transparent;
 }

diff --git a/frontend/src/utils/schemaValidation.js b/frontend/src/utils/schemaValidation.js
@@ -7,7 +7,7 @@ export default function generateSchema(pipeline) {
       schemaFields[key] = z.any().optional();
     } else if (typeof value === "object" && !Array.isArray(value)) {
       schemaFields[key] = generateSchema(value);
-    } else if (key === "value") {
+    } else if (key === "value" && !Array.isArray(value)) {
       schemaFields[key] = z
         .any()
         .refine((val) => val.replace(/['"]+/g, "").trim().length !== 0, {