File block (#222)

* "Added FolderBlock component and updated conditional statement to render it for 'folder' type" * "Added FolderBlock component and updated conditional statement to render it for 'folder' type" * Added MultiFileBlock component * Added MultiFileBlock component * updates on multi-file block * Update folder upload block specs and BlockGenerator component * Updated email and github of Zetane * try 1 * try 1 * folder and multi-file works, with minor bugs * folder upload block shows it's folders name. * folder-upload and multi-files upload working fine. * got rid of localStorage --------- Co-authored-by: Jon Magoon <[email protected]>
zetane · Oct 22, 2024 · 48f6047 · 48f6047
1 parent 45fc61a
commit 48f6047
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 42 deletions.
diff --git a/frontend/server/pipelineSerialization.js b/frontend/server/pipelineSerialization.js
@@ -6,17 +6,17 @@ import {
   BLOCK_SPECS_FILE_NAME,
   PIPELINE_SPECS_FILE_NAME,
 } from "../src/utils/constants";
-import { setDifference } from "../utils/set.js";
+// import { setDifference } from "../utils/set.js";
 import {
-  fileExists,
+  // fileExists,
   filterDirectories,
-  readJsonToObject,
+  // readJsonToObject,
 } from "./fileSystem.js";
 import { checkAndUpload, checkAndCopy, uploadDirectory } from "./s3.js";
 import {
   createExecution,
   getBuildContextStatus,
-  getPipelinesByUuid,
+  // getPipelinesByUuid,
 } from "./anvil";
 import { logger } from "./logger";
 import { computePipelineMerkleTree } from "./merkle";
@@ -60,19 +60,19 @@ export async function copyPipeline(pipelineSpecs, fromDir, toDir) {
 
   const fromBlockIndex = await getBlockIndex([bufferPath]);
 
-  let toBlockIndex = {};
-  if (await fileExists(writePipelineDirectory)) {
-    toBlockIndex = await getBlockIndex([writePipelineDirectory]);
-  } else {
-    await fs.mkdir(writePipelineDirectory, { recursive: true });
-  }
+  // let toBlockIndex = {};
+  // if (await fileExists(writePipelineDirectory)) {
+  //   toBlockIndex = await getBlockIndex([writePipelineDirectory]);
+  // } else {
+  //   await fs.mkdir(writePipelineDirectory, { recursive: true });
+  // }
 
   // Gets pipeline specs from the specs coming from the graph
   // Submitted by the client
   const newPipelineBlocks = getPipelineBlocks(pipelineSpecs);
-  const existingPipelineBlocks = (await fileExists(pipelineSpecsPath))
-    ? await readPipelineBlocks(pipelineSpecsPath)
-    : new Set();
+  // const existingPipelineBlocks = (await fileExists(pipelineSpecsPath))
+  //   ? await readPipelineBlocks(pipelineSpecsPath)
+  //   : new Set();
 
   for (const key of Array.from(newPipelineBlocks)) {
     const newBlockPath = path.join(writePipelineDirectory, key);
@@ -153,10 +153,10 @@ async function getBlocksInDirectory(directory) {
   return directories;
 }
 
-async function readPipelineBlocks(specsPath) {
-  const specs = await readJsonToObject(specsPath);
-  return getPipelineBlocks(specs);
-}
+// async function readPipelineBlocks(specsPath) {
+//   const specs = await readJsonToObject(specsPath);
+//   return getPipelineBlocks(specs);
+// }
 
 export async function removeBlock(blockId, pipelinePath) {
   const blockPath = await getPipelineBlockPath(pipelinePath, blockId);
@@ -234,6 +234,35 @@ async function uploadBlocks(
             param.value = `"${fileName}"`;
             param.type = "blob";
           }
+        } else if (param.type === "folder" || param.type === "file[]") {
+          try {
+            const cleanedValue = param.value.replace(/\\/g, "\\\\");
+            const fileNames = [];
+            const filePaths = JSON.parse(cleanedValue);
+
+            for (const filePath of filePaths) {
+              // console.log("Uploading file:", filePath); // Debugging log
+              const fileName = path.basename(filePath);
+              fileNames.push(fileName);
+              const awsKey = `${pipelineId}/${executionId}/${fileName}`;
+
+              if (filePath && filePath.trim()) {
+                await checkAndUpload(awsKey, filePath, anvilConfiguration);
+                // console.log(`Uploaded: ${fileName} to ${awsKey}`); // which file uploaded.
+              } else {
+                // log invalid paths.
+                console.error("Invalid file path:", filePath);
+              }
+            }
+            // Preserve the original value and update type
+            param.value =
+              fileNames.length > 0
+                ? `["${fileNames.join('", "')}"]`
+                : param.value;
+            param.type = "blob";
+          } catch (error) {
+            console.error("Error processing folder:", error);
+          }
         } else if (param.type == "blob") {
           const copyKey = param.value;
           const fileName = param.value.split("/").at(-1);

diff --git a/frontend/src/components/ui/blockGenerator/BlockGenerator.jsx b/frontend/src/components/ui/blockGenerator/BlockGenerator.jsx
@@ -142,7 +142,11 @@ const BlockGenerator = ({
         history={history}
       />
     );
-  } else if (type == "file[]" || type == "multiFile") {
+  } else if (
+    type == "file[]" ||
+    type == "multiFile" ||
+    block.information.id == "multi-file-upload"
+  ) {
     content = (
       <MultiFileBlock
         blockId={id}

diff --git a/frontend/src/components/ui/blockGenerator/Folder-uploadBlock.jsx b/frontend/src/components/ui/blockGenerator/Folder-uploadBlock.jsx
@@ -1,55 +1,70 @@
 import { useEffect, useRef, useState } from "react";
-import { trimQuotes } from "@/utils/blockUtils";
 
 export const FolderBlock = ({ blockId, block, setFocusAction, history }) => {
   const fileInput = useRef();
   const [renderPath, setRenderPath] = useState(null);
+  const [folderName, setFolderName] = useState(
+    block?.action?.parameters["folderName"]?.value || null,
+  );
+  const [filePaths, setFilePaths] = useState(() => {
+    const existingPaths = block?.action?.parameters["path"]?.value || "[]";
+    return JSON.parse(existingPaths);
+  });
 
   useEffect(() => {
-    const type = block?.action?.parameters["path"]?.type;
-    if (history && type !== "fileLoad") {
-      let fileName = block?.action?.parameters["path"]?.value;
-      const fileSplit = fileName.split("/");
-      if (fileSplit.length > 1) {
-        fileName = fileSplit.at(-1);
-      }
-      fileName = trimQuotes(fileName);
-      const s3Url = history + "/" + fileName;
+    if (history && filePaths.length > 0 && folderName) {
+      const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`;
       setFocusAction((draft) => {
-        draft.data[blockId].action.parameters["path"].value = s3Url;
-        draft.data[blockId].action.parameters["path"].type = "blob";
+        draft.data[blockId].action.parameters["path"].type = "folder";
+        draft.data[blockId].action.parameters["path"].value = formattedValue;
+        draft.data[blockId].action.parameters["folderName"] = {
+          type: "string",
+          value: folderName,
+        };
       });
-      setRenderPath(fileName);
+      setRenderPath(folderName);
     }
-  }, [block]);
+  }, [block, folderName, filePaths]);
 
   const processFiles = (files) => {
     const filePaths = [];
     const readDirectory = (dir, path = "") => {
       for (let i = 0; i < dir.length; i++) {
         const file = dir[i];
-        const fullPath = path ? `${path}/${file.name}` : file.name;
+        const fullPath = path ? `${path}/${file.path}` : file.path;
         if (file.webkitDirectory) {
           readDirectory(file.children, fullPath);
         } else {
           filePaths.push(fullPath);
         }
       }
     };
-
     readDirectory(files, "");
     return filePaths;
   };
 
   const loadFiles = (e) => {
     const files = Array.from(fileInput.current.files);
     const filePaths = processFiles(files);
-    const value = filePaths.join(", ");
+    const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`;
+
+    const firstFilePath = filePaths[0];
+    const pathSegments = firstFilePath.split(/[/\\]/);
+    const extractedFolderName = pathSegments[pathSegments.length - 2];
+
+    setFolderName(extractedFolderName);
+    setFilePaths(filePaths);
+
     setFocusAction((draft) => {
-      draft.data[blockId].action.parameters["path"].value = value;
+      draft.data[blockId].action.parameters["path"].value = formattedValue;
       draft.data[blockId].action.parameters["path"].type = "folder";
+      draft.data[blockId].action.parameters["folderName"] = {
+        type: "string",
+        value: extractedFolderName,
+      };
     });
-    setRenderPath(value);
+
+    setRenderPath(extractedFolderName);
   };
 
   return (

diff --git a/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx b/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx
@@ -22,13 +22,32 @@ export const MultiFileBlock = ({ blockId, block, setFocusAction, history }) => {
     }
   }, [block]);
 
+  const processFiles = (files) => {
+    const filePaths = [];
+    const readDirectory = (dir, path = "") => {
+      for (let i = 0; i < dir.length; i++) {
+        const file = dir[i];
+        const fullPath = path ? `${path}/${file.path}` : file.path;
+        if (file.webkitDirectory) {
+          readDirectory(file.children, fullPath);
+        } else {
+          filePaths.push(fullPath);
+        }
+      }
+    };
+    readDirectory(files, "");
+    return filePaths;
+  };
+
   const loadFiles = (e) => {
     const files = Array.from(fileInput.current.files);
-    const filePaths = files.map((file) => file.name);
-    const value = filePaths.join(", ");
-
+    // console.log("FILES: " ,files)
+    const filePaths = processFiles(files);
+    // console.log("filePaths:" , filePaths)
+    const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`;
+    // console.log("formatted value: " , formattedValue)
     setFocusAction((draft) => {
-      draft.data[blockId].action.parameters["files"].value = value;
+      draft.data[blockId].action.parameters["files"].value = formattedValue; // Update to formatted value
       draft.data[blockId].action.parameters["files"].type = "file[]";
     });
 
@@ -57,7 +76,6 @@ export const MultiFileBlock = ({ blockId, block, setFocusAction, history }) => {
         style={{
           // for better visibility.
           color: "#ffffff",
-          // backgroundColor: "#333333",
         }}
       />
     </div>